2024-11-09 22:39:34,452 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 22:39:34,474 main DEBUG Took 0.017773 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 22:39:34,475 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 22:39:34,476 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 22:39:34,477 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 22:39:34,479 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,493 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 22:39:34,512 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,516 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,517 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,518 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,519 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,520 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,522 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,523 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,524 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,525 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,526 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,527 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,528 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,528 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 22:39:34,529 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,530 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 22:39:34,532 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 22:39:34,533 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 22:39:34,540 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 22:39:34,540 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 22:39:34,542 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 22:39:34,543 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 22:39:34,554 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 22:39:34,557 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 22:39:34,561 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 22:39:34,563 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 22:39:34,566 main DEBUG createAppenders(={Console}) 2024-11-09 22:39:34,568 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-09 22:39:34,570 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 22:39:34,571 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-09 22:39:34,571 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 22:39:34,572 main DEBUG OutputStream closed 2024-11-09 22:39:34,572 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 22:39:34,572 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 22:39:34,573 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-09 22:39:34,674 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 22:39:34,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 22:39:34,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 22:39:34,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 22:39:34,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 22:39:34,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 22:39:34,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 22:39:34,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 22:39:34,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 22:39:34,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 22:39:34,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 22:39:34,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 22:39:34,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 22:39:34,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 22:39:34,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 22:39:34,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 22:39:34,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 22:39:34,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 22:39:34,693 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 22:39:34,694 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-09 22:39:34,694 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 22:39:34,695 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-09T22:39:35,043 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09 2024-11-09 22:39:35,048 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 22:39:35,050 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T22:39:35,062 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-09T22:39:35,134 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T22:39:35,164 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5, deleteOnExit=true 2024-11-09T22:39:35,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T22:39:35,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/test.cache.data in system properties and HBase conf 2024-11-09T22:39:35,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T22:39:35,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/hadoop.log.dir in system properties and HBase conf 2024-11-09T22:39:35,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T22:39:35,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T22:39:35,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T22:39:35,328 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T22:39:35,454 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T22:39:35,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T22:39:35,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T22:39:35,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T22:39:35,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T22:39:35,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T22:39:35,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T22:39:35,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T22:39:35,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T22:39:35,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T22:39:35,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/nfs.dump.dir in system properties and HBase conf 2024-11-09T22:39:35,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/java.io.tmpdir in system properties and HBase conf 2024-11-09T22:39:35,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T22:39:35,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T22:39:35,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T22:39:36,635 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T22:39:36,730 INFO [Time-limited test {}] log.Log(170): Logging initialized @3323ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T22:39:36,814 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T22:39:36,903 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T22:39:36,936 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T22:39:36,937 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T22:39:36,939 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T22:39:36,958 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T22:39:36,965 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/hadoop.log.dir/,AVAILABLE} 2024-11-09T22:39:36,967 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T22:39:37,204 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/java.io.tmpdir/jetty-localhost-43921-hadoop-hdfs-3_4_1-tests_jar-_-any-17944563509787676405/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T22:39:37,212 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43921} 2024-11-09T22:39:37,213 INFO [Time-limited test {}] server.Server(415): Started @3807ms 2024-11-09T22:39:37,818 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T22:39:37,826 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T22:39:37,828 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T22:39:37,828 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T22:39:37,828 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T22:39:37,829 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/hadoop.log.dir/,AVAILABLE} 2024-11-09T22:39:37,830 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T22:39:37,945 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/java.io.tmpdir/jetty-localhost-35643-hadoop-hdfs-3_4_1-tests_jar-_-any-16487216609201103214/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T22:39:37,946 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:35643} 2024-11-09T22:39:37,946 INFO [Time-limited test {}] server.Server(415): Started @4540ms 2024-11-09T22:39:38,000 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T22:39:39,120 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/data/data1/current/BP-454095681-172.17.0.2-1731191976142/current, will proceed with Du for space computation calculation, 2024-11-09T22:39:39,120 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/data/data2/current/BP-454095681-172.17.0.2-1731191976142/current, will proceed with Du for space computation calculation, 2024-11-09T22:39:39,151 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T22:39:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3e2c079bd88fcd07 with lease ID 0xea4470012fa512a0: Processing first storage report for DS-561c947f-dee8-478d-a597-409575083358 from datanode DatanodeRegistration(127.0.0.1:41787, datanodeUuid=67278256-a007-410b-a3ce-fada866abf51, infoPort=45935, infoSecurePort=0, ipcPort=43339, storageInfo=lv=-57;cid=testClusterID;nsid=1398308083;c=1731191976142) 2024-11-09T22:39:39,203 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e2c079bd88fcd07 with lease ID 0xea4470012fa512a0: from storage DS-561c947f-dee8-478d-a597-409575083358 node DatanodeRegistration(127.0.0.1:41787, datanodeUuid=67278256-a007-410b-a3ce-fada866abf51, infoPort=45935, infoSecurePort=0, ipcPort=43339, storageInfo=lv=-57;cid=testClusterID;nsid=1398308083;c=1731191976142), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-09T22:39:39,204 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3e2c079bd88fcd07 with lease ID 0xea4470012fa512a0: Processing first storage report for DS-d70578ae-b45f-452a-8594-23b717117ddb from datanode DatanodeRegistration(127.0.0.1:41787, datanodeUuid=67278256-a007-410b-a3ce-fada866abf51, infoPort=45935, infoSecurePort=0, ipcPort=43339, storageInfo=lv=-57;cid=testClusterID;nsid=1398308083;c=1731191976142) 2024-11-09T22:39:39,205 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e2c079bd88fcd07 with lease ID 0xea4470012fa512a0: from storage DS-d70578ae-b45f-452a-8594-23b717117ddb node DatanodeRegistration(127.0.0.1:41787, datanodeUuid=67278256-a007-410b-a3ce-fada866abf51, infoPort=45935, infoSecurePort=0, ipcPort=43339, storageInfo=lv=-57;cid=testClusterID;nsid=1398308083;c=1731191976142), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T22:39:39,248 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09 2024-11-09T22:39:39,346 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/zookeeper_0, clientPort=49842, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T22:39:39,360 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49842 2024-11-09T22:39:39,375 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T22:39:39,379 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T22:39:39,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741825_1001 (size=7) 2024-11-09T22:39:40,028 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 with version=8 2024-11-09T22:39:40,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/hbase-staging 2024-11-09T22:39:40,114 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T22:39:40,313 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9d8c37854f6e:0 server-side Connection retries=45 2024-11-09T22:39:40,321 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T22:39:40,322 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T22:39:40,326 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T22:39:40,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T22:39:40,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T22:39:40,461 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T22:39:40,535 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T22:39:40,544 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T22:39:40,548 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T22:39:40,570 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 989 (auto-detected) 2024-11-09T22:39:40,571 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T22:39:40,591 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34651 2024-11-09T22:39:40,618 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34651 connecting to ZooKeeper ensemble=127.0.0.1:49842 2024-11-09T22:39:40,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:346510x0, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T22:39:40,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34651-0x101230acffd0000 connected 2024-11-09T22:39:40,894 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T22:39:40,898 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T22:39:40,909 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T22:39:40,912 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696, hbase.cluster.distributed=false 2024-11-09T22:39:40,938 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T22:39:40,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34651 2024-11-09T22:39:40,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34651 2024-11-09T22:39:40,946 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34651 2024-11-09T22:39:40,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34651 2024-11-09T22:39:40,951 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34651 2024-11-09T22:39:41,062 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9d8c37854f6e:0 server-side Connection retries=45 2024-11-09T22:39:41,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T22:39:41,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T22:39:41,064 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T22:39:41,064 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T22:39:41,065 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T22:39:41,067 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T22:39:41,071 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T22:39:41,072 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35101 2024-11-09T22:39:41,074 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35101 connecting to ZooKeeper ensemble=127.0.0.1:49842 2024-11-09T22:39:41,076 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T22:39:41,079 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T22:39:41,101 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:351010x0, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T22:39:41,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351010x0, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T22:39:41,103 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35101-0x101230acffd0001 connected 2024-11-09T22:39:41,106 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T22:39:41,116 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T22:39:41,119 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T22:39:41,125 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T22:39:41,126 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35101 2024-11-09T22:39:41,126 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35101 2024-11-09T22:39:41,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35101 2024-11-09T22:39:41,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35101 2024-11-09T22:39:41,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35101 2024-11-09T22:39:41,161 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9d8c37854f6e:34651 2024-11-09T22:39:41,162 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:41,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T22:39:41,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T22:39:41,177 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:41,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T22:39:41,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:41,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:41,214 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T22:39:41,215 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9d8c37854f6e,34651,1731191980160 from backup master directory 2024-11-09T22:39:41,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:41,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T22:39:41,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T22:39:41,225 WARN [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T22:39:41,225 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:41,227 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T22:39:41,229 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T22:39:41,287 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/hbase.id] with ID: c19612e1-c4b9-4038-b012-a363c57ebb38 2024-11-09T22:39:41,287 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/.tmp/hbase.id 2024-11-09T22:39:41,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741826_1002 (size=42) 2024-11-09T22:39:41,703 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/.tmp/hbase.id]:[hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/hbase.id] 2024-11-09T22:39:41,756 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T22:39:41,762 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T22:39:41,784 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-09T22:39:41,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:41,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:41,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741827_1003 (size=196) 2024-11-09T22:39:42,273 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:39:42,275 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T22:39:42,290 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:42,294 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T22:39:42,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741828_1004 (size=1189) 2024-11-09T22:39:42,348 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store 2024-11-09T22:39:42,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741829_1005 (size=34) 2024-11-09T22:39:42,773 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T22:39:42,776 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:39:42,778 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T22:39:42,778 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:39:42,778 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:39:42,781 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T22:39:42,781 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:39:42,781 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:39:42,783 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731191982778Disabling compacts and flushes for region at 1731191982778Disabling writes for close at 1731191982781 (+3 ms)Writing region close event to WAL at 1731191982781Closed at 1731191982781 2024-11-09T22:39:42,785 WARN [master/9d8c37854f6e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/.initializing 2024-11-09T22:39:42,786 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/WALs/9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:42,794 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T22:39:42,812 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d8c37854f6e%2C34651%2C1731191980160, suffix=, logDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/WALs/9d8c37854f6e,34651,1731191980160, archiveDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/oldWALs, maxLogs=10 2024-11-09T22:39:42,836 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/WALs/9d8c37854f6e,34651,1731191980160/9d8c37854f6e%2C34651%2C1731191980160.1731191982818, exclude list is [], retry=0 2024-11-09T22:39:42,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-561c947f-dee8-478d-a597-409575083358,DISK] 2024-11-09T22:39:42,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T22:39:42,896 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/WALs/9d8c37854f6e,34651,1731191980160/9d8c37854f6e%2C34651%2C1731191980160.1731191982818 2024-11-09T22:39:42,897 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45935:45935)] 2024-11-09T22:39:42,898 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:39:42,899 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:39:42,903 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,904 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,941 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T22:39:42,968 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:42,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:42,971 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T22:39:42,974 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:42,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:39:42,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,979 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T22:39:42,979 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:42,980 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:39:42,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,985 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T22:39:42,985 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:42,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:39:42,987 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,991 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,993 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:42,999 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:43,000 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:43,005 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T22:39:43,010 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T22:39:43,016 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:39:43,018 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67792761, jitterRate=0.01019085943698883}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T22:39:43,025 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731191982917Initializing all the Stores at 1731191982919 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191982919Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191982920 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191982920Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191982920Cleaning up temporary data from old regions at 1731191983000 (+80 ms)Region opened successfully at 1731191983025 (+25 ms) 2024-11-09T22:39:43,027 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T22:39:43,063 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a2e083d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9d8c37854f6e/172.17.0.2:0 2024-11-09T22:39:43,092 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T22:39:43,107 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T22:39:43,107 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T22:39:43,110 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T22:39:43,113 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-09T22:39:43,119 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T22:39:43,119 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T22:39:43,144 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T22:39:43,153 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T22:39:43,213 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T22:39:43,217 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T22:39:43,219 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T22:39:43,229 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T22:39:43,231 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T22:39:43,236 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T22:39:43,245 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T22:39:43,248 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T22:39:43,262 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T22:39:43,283 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T22:39:43,290 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T22:39:43,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T22:39:43,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T22:39:43,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:43,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:43,311 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9d8c37854f6e,34651,1731191980160, sessionid=0x101230acffd0000, setting cluster-up flag (Was=false) 2024-11-09T22:39:43,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:43,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:43,399 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T22:39:43,402 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:43,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:43,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:43,482 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T22:39:43,486 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:43,495 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T22:39:43,547 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(746): ClusterId : c19612e1-c4b9-4038-b012-a363c57ebb38 2024-11-09T22:39:43,550 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T22:39:43,568 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T22:39:43,577 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T22:39:43,584 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T22:39:43,585 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T22:39:43,585 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T22:39:43,591 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9d8c37854f6e,34651,1731191980160 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T22:39:43,597 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T22:39:43,597 DEBUG [RS:0;9d8c37854f6e:35101 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b36b7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9d8c37854f6e/172.17.0.2:0 2024-11-09T22:39:43,598 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9d8c37854f6e:0, corePoolSize=5, maxPoolSize=5 2024-11-09T22:39:43,598 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9d8c37854f6e:0, corePoolSize=5, maxPoolSize=5 2024-11-09T22:39:43,599 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9d8c37854f6e:0, corePoolSize=5, maxPoolSize=5 2024-11-09T22:39:43,599 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9d8c37854f6e:0, corePoolSize=5, maxPoolSize=5 2024-11-09T22:39:43,599 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9d8c37854f6e:0, corePoolSize=10, maxPoolSize=10 2024-11-09T22:39:43,599 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,599 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9d8c37854f6e:0, corePoolSize=2, maxPoolSize=2 2024-11-09T22:39:43,600 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,604 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731192013604 2024-11-09T22:39:43,607 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T22:39:43,608 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T22:39:43,612 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T22:39:43,612 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9d8c37854f6e:35101 2024-11-09T22:39:43,613 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T22:39:43,613 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T22:39:43,613 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T22:39:43,615 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T22:39:43,615 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T22:39:43,616 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T22:39:43,616 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T22:39:43,616 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T22:39:43,614 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,619 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d8c37854f6e,34651,1731191980160 with port=35101, startcode=1731191981027 2024-11-09T22:39:43,623 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:43,623 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T22:39:43,623 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T22:39:43,624 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T22:39:43,624 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T22:39:43,627 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T22:39:43,627 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T22:39:43,629 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9d8c37854f6e:0:becomeActiveMaster-HFileCleaner.large.0-1731191983629,5,FailOnTimeoutGroup] 2024-11-09T22:39:43,631 DEBUG [RS:0;9d8c37854f6e:35101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T22:39:43,646 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9d8c37854f6e:0:becomeActiveMaster-HFileCleaner.small.0-1731191983629,5,FailOnTimeoutGroup] 2024-11-09T22:39:43,647 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,647 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T22:39:43,648 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,649 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741831_1007 (size=1321) 2024-11-09T22:39:43,653 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T22:39:43,654 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 2024-11-09T22:39:43,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741832_1008 (size=32) 2024-11-09T22:39:43,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:39:43,695 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39645, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T22:39:43,700 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T22:39:43,703 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34651 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:43,706 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34651 {}] master.ServerManager(517): Registering regionserver=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:43,714 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T22:39:43,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:43,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:43,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T22:39:43,721 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T22:39:43,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:43,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:43,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T22:39:43,724 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 2024-11-09T22:39:43,724 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46069 2024-11-09T22:39:43,724 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T22:39:43,727 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T22:39:43,727 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:43,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:43,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T22:39:43,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T22:39:43,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:43,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:43,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T22:39:43,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740 2024-11-09T22:39:43,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740 2024-11-09T22:39:43,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T22:39:43,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T22:39:43,744 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:39:43,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T22:39:43,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T22:39:43,749 DEBUG [RS:0;9d8c37854f6e:35101 {}] zookeeper.ZKUtil(111): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:43,750 WARN [RS:0;9d8c37854f6e:35101 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T22:39:43,750 INFO [RS:0;9d8c37854f6e:35101 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T22:39:43,750 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/WALs/9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:43,758 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:39:43,758 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9d8c37854f6e,35101,1731191981027] 2024-11-09T22:39:43,760 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66205436, jitterRate=-0.0134621262550354}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:39:43,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731191983694Initializing all the Stores at 1731191983697 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191983697Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191983698 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191983699 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191983699Cleaning up temporary data from old regions at 1731191983743 (+44 ms)Region opened successfully at 1731191983764 (+21 ms) 2024-11-09T22:39:43,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T22:39:43,765 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T22:39:43,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T22:39:43,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T22:39:43,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T22:39:43,767 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T22:39:43,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731191983765Disabling compacts and flushes for region at 1731191983765Disabling writes for close at 1731191983766 (+1 ms)Writing region close event to WAL at 1731191983767 (+1 ms)Closed at 1731191983767 2024-11-09T22:39:43,772 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T22:39:43,772 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T22:39:43,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T22:39:43,784 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T22:39:43,792 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T22:39:43,797 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T22:39:43,806 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T22:39:43,813 INFO [RS:0;9d8c37854f6e:35101 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T22:39:43,813 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,814 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T22:39:43,821 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T22:39:43,823 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,824 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,824 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,824 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,824 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,824 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,825 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9d8c37854f6e:0, corePoolSize=2, maxPoolSize=2 2024-11-09T22:39:43,825 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,825 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,825 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,825 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,825 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,826 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9d8c37854f6e:0, corePoolSize=1, maxPoolSize=1 2024-11-09T22:39:43,826 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9d8c37854f6e:0, corePoolSize=3, maxPoolSize=3 2024-11-09T22:39:43,826 DEBUG [RS:0;9d8c37854f6e:35101 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0, corePoolSize=3, maxPoolSize=3 2024-11-09T22:39:43,827 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,828 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,828 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,828 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,828 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,828 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d8c37854f6e,35101,1731191981027-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T22:39:43,853 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T22:39:43,854 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d8c37854f6e,35101,1731191981027-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,855 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,855 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.Replication(171): 9d8c37854f6e,35101,1731191981027 started 2024-11-09T22:39:43,879 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:43,880 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1482): Serving as 9d8c37854f6e,35101,1731191981027, RpcServer on 9d8c37854f6e/172.17.0.2:35101, sessionid=0x101230acffd0001 2024-11-09T22:39:43,881 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T22:39:43,881 DEBUG [RS:0;9d8c37854f6e:35101 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:43,881 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d8c37854f6e,35101,1731191981027' 2024-11-09T22:39:43,881 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T22:39:43,882 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T22:39:43,883 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T22:39:43,883 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T22:39:43,883 DEBUG [RS:0;9d8c37854f6e:35101 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:43,884 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d8c37854f6e,35101,1731191981027' 2024-11-09T22:39:43,884 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T22:39:43,886 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T22:39:43,887 DEBUG [RS:0;9d8c37854f6e:35101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T22:39:43,887 INFO [RS:0;9d8c37854f6e:35101 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T22:39:43,887 INFO [RS:0;9d8c37854f6e:35101 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T22:39:43,948 WARN [9d8c37854f6e:34651 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T22:39:43,992 INFO [RS:0;9d8c37854f6e:35101 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T22:39:43,996 INFO [RS:0;9d8c37854f6e:35101 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d8c37854f6e%2C35101%2C1731191981027, suffix=, logDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/WALs/9d8c37854f6e,35101,1731191981027, archiveDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/oldWALs, maxLogs=32 2024-11-09T22:39:44,013 DEBUG [RS:0;9d8c37854f6e:35101 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/WALs/9d8c37854f6e,35101,1731191981027/9d8c37854f6e%2C35101%2C1731191981027.1731191984000, exclude list is [], retry=0 2024-11-09T22:39:44,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-561c947f-dee8-478d-a597-409575083358,DISK] 2024-11-09T22:39:44,023 INFO [RS:0;9d8c37854f6e:35101 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/WALs/9d8c37854f6e,35101,1731191981027/9d8c37854f6e%2C35101%2C1731191981027.1731191984000 2024-11-09T22:39:44,023 DEBUG [RS:0;9d8c37854f6e:35101 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45935:45935)] 2024-11-09T22:39:44,202 DEBUG [9d8c37854f6e:34651 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T22:39:44,216 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:44,221 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9d8c37854f6e,35101,1731191981027, state=OPENING 2024-11-09T22:39:44,282 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T22:39:44,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:44,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:39:44,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T22:39:44,291 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T22:39:44,292 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T22:39:44,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:39:44,473 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T22:39:44,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T22:39:44,487 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T22:39:44,487 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T22:39:44,488 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T22:39:44,491 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d8c37854f6e%2C35101%2C1731191981027.meta, suffix=.meta, logDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/WALs/9d8c37854f6e,35101,1731191981027, archiveDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/oldWALs, maxLogs=32 2024-11-09T22:39:44,505 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/WALs/9d8c37854f6e,35101,1731191981027/9d8c37854f6e%2C35101%2C1731191981027.meta.1731191984493.meta, exclude list is [], retry=0 2024-11-09T22:39:44,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-561c947f-dee8-478d-a597-409575083358,DISK] 2024-11-09T22:39:44,511 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/WALs/9d8c37854f6e,35101,1731191981027/9d8c37854f6e%2C35101%2C1731191981027.meta.1731191984493.meta 2024-11-09T22:39:44,512 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45935:45935)] 2024-11-09T22:39:44,512 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:39:44,514 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T22:39:44,516 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T22:39:44,520 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T22:39:44,525 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T22:39:44,525 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:39:44,526 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T22:39:44,526 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T22:39:44,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T22:39:44,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T22:39:44,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:44,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:44,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T22:39:44,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T22:39:44,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:44,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:44,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T22:39:44,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T22:39:44,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:44,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:44,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T22:39:44,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T22:39:44,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:44,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T22:39:44,546 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T22:39:44,548 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740 2024-11-09T22:39:44,552 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740 2024-11-09T22:39:44,554 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T22:39:44,554 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T22:39:44,555 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:39:44,559 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T22:39:44,561 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59508453, jitterRate=-0.11325494945049286}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:39:44,562 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T22:39:44,563 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731191984526Writing region info on filesystem at 1731191984526Initializing all the Stores at 1731191984529 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191984529Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191984529Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191984529Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731191984529Cleaning up temporary data from old regions at 1731191984554 (+25 ms)Running coprocessor post-open hooks at 1731191984562 (+8 ms)Region opened successfully at 1731191984563 (+1 ms) 2024-11-09T22:39:44,570 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731191984465 2024-11-09T22:39:44,582 DEBUG [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T22:39:44,583 INFO [RS_OPEN_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T22:39:44,585 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:44,587 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9d8c37854f6e,35101,1731191981027, state=OPEN 2024-11-09T22:39:44,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T22:39:44,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T22:39:44,642 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T22:39:44,642 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T22:39:44,642 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:44,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T22:39:44,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9d8c37854f6e,35101,1731191981027 in 348 msec 2024-11-09T22:39:44,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T22:39:44,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 873 msec 2024-11-09T22:39:44,662 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T22:39:44,662 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T22:39:44,682 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:44,685 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:44,712 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:44,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58001, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:44,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2110 sec 2024-11-09T22:39:44,738 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731191984738, completionTime=-1 2024-11-09T22:39:44,740 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T22:39:44,740 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T22:39:44,768 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-09T22:39:44,768 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731192044768 2024-11-09T22:39:44,768 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731192104768 2024-11-09T22:39:44,768 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-09T22:39:44,770 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d8c37854f6e,34651,1731191980160-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:44,771 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d8c37854f6e,34651,1731191980160-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:44,771 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d8c37854f6e,34651,1731191980160-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:44,773 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9d8c37854f6e:34651, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:44,774 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:44,775 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:44,792 DEBUG [master/9d8c37854f6e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T22:39:44,811 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.586sec 2024-11-09T22:39:44,813 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T22:39:44,814 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T22:39:44,816 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T22:39:44,816 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T22:39:44,817 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T22:39:44,818 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d8c37854f6e,34651,1731191980160-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T22:39:44,818 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d8c37854f6e,34651,1731191980160-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T22:39:44,846 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T22:39:44,847 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T22:39:44,848 INFO [master/9d8c37854f6e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d8c37854f6e,34651,1731191980160-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T22:39:44,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:44,862 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T22:39:44,863 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T22:39:44,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:44,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:44,880 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5abd05e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:44,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:44,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:44,901 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:44,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:44,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:44,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:44,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:44,955 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=19016 2024-11-09T22:39:44,966 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T22:39:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:44,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79901794 2024-11-09T22:39:44,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T22:39:44,981 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T22:39:44,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:39:44,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:39:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T22:39:44,999 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T22:39:45,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-09T22:39:45,002 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:45,004 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T22:39:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:39:45,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741835_1011 (size=959) 2024-11-09T22:39:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:39:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:39:45,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 2024-11-09T22:39:45,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741836_1012 (size=53) 2024-11-09T22:39:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:39:45,835 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:39:45,835 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1f18320792a575fd182c9198939e0c16, disabling compactions & flushes 2024-11-09T22:39:45,835 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:45,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:45,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. after waiting 0 ms 2024-11-09T22:39:45,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:45,836 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:45,836 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1f18320792a575fd182c9198939e0c16: Waiting for close lock at 1731191985835Disabling compacts and flushes for region at 1731191985835Disabling writes for close at 1731191985836 (+1 ms)Writing region close event to WAL at 1731191985836Closed at 1731191985836 2024-11-09T22:39:45,839 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T22:39:45,843 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731191985839"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731191985839"}]},"ts":"1731191985839"} 2024-11-09T22:39:45,847 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T22:39:45,850 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T22:39:45,853 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731191985850"}]},"ts":"1731191985850"} 2024-11-09T22:39:45,859 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T22:39:45,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, ASSIGN}] 2024-11-09T22:39:45,864 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, ASSIGN 2024-11-09T22:39:45,867 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, ASSIGN; state=OFFLINE, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=false 2024-11-09T22:39:46,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1f18320792a575fd182c9198939e0c16, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:46,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, ASSIGN because future has completed 2024-11-09T22:39:46,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:39:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:39:46,190 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:46,190 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:39:46,191 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,191 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:39:46,191 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,191 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,194 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,196 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f18320792a575fd182c9198939e0c16 columnFamilyName A 2024-11-09T22:39:46,196 DEBUG [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:46,197 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] regionserver.HStore(327): Store=1f18320792a575fd182c9198939e0c16/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:39:46,197 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,200 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f18320792a575fd182c9198939e0c16 columnFamilyName B 2024-11-09T22:39:46,200 DEBUG [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:46,201 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] regionserver.HStore(327): Store=1f18320792a575fd182c9198939e0c16/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:39:46,201 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,204 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f18320792a575fd182c9198939e0c16 columnFamilyName C 2024-11-09T22:39:46,204 DEBUG [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:39:46,205 INFO [StoreOpener-1f18320792a575fd182c9198939e0c16-1 {}] regionserver.HStore(327): Store=1f18320792a575fd182c9198939e0c16/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:39:46,205 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,207 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,208 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,210 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,210 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,211 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:39:46,213 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,216 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:39:46,217 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1f18320792a575fd182c9198939e0c16; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67090657, jitterRate=-2.713054418563843E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:39:46,218 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:46,219 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1f18320792a575fd182c9198939e0c16: Running coprocessor pre-open hook at 1731191986191Writing region info on filesystem at 1731191986191Initializing all the Stores at 1731191986193 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191986193Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191986194 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731191986194Cleaning up temporary data from old regions at 1731191986210 (+16 ms)Running coprocessor post-open hooks at 1731191986218 (+8 ms)Region opened successfully at 1731191986219 (+1 ms) 2024-11-09T22:39:46,221 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., pid=6, masterSystemTime=1731191986183 2024-11-09T22:39:46,224 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:46,224 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:46,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1f18320792a575fd182c9198939e0c16, regionState=OPEN, openSeqNum=2, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:46,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:39:46,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T22:39:46,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 in 206 msec 2024-11-09T22:39:46,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T22:39:46,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, ASSIGN in 375 msec 2024-11-09T22:39:46,243 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T22:39:46,243 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731191986243"}]},"ts":"1731191986243"} 2024-11-09T22:39:46,247 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T22:39:46,249 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T22:39:46,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2580 sec 2024-11-09T22:39:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T22:39:47,159 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:39:47,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e68a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253618fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,173 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dec8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,175 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,182 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T22:39:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d8c37854f6e,34651,1731191980160 2024-11-09T22:39:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1df1000 2024-11-09T22:39:47,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T22:39:47,189 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T22:39:47,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,197 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,200 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,204 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,208 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,213 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,216 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,225 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,229 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df7e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,233 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4098c4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,238 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ccd238a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f410f5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f6ca0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,256 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74deee7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,260 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e021fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,264 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@905b4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526369d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:39:47,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:39:47,272 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:39:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:39:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:39:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ce558b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:39:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:39:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:39:47,275 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:39:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280a2bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:39:47,283 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,295 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:39:47,311 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-09T22:39:47,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:39:47,342 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,343 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T22:39:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:39:47,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,348 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:39:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,355 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,356 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,359 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,385 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:39:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:39:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:39:47,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:39:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:39:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T22:39:47,457 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:47,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T22:39:47,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T22:39:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:47,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/e9e167697c7b424e930e1d48813999fa is 50, key is test_row_0/A:col10/1731191987384/Put/seqid=0 2024-11-09T22:39:47,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741837_1013 (size=12001) 2024-11-09T22:39:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192047623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192047618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/e9e167697c7b424e930e1d48813999fa 2024-11-09T22:39:47,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192047626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192047628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192047632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T22:39:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T22:39:47,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:47,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:47,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:47,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/bdd93c98994a4a74867ad5a203c49fa3 is 50, key is test_row_0/B:col10/1731191987384/Put/seqid=0 2024-11-09T22:39:47,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192047777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192047777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192047777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192047778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192047778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:47,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741838_1014 (size=12001) 2024-11-09T22:39:47,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/bdd93c98994a4a74867ad5a203c49fa3 2024-11-09T22:39:47,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/4c23aa3ca0ba4734a308b18e802abac0 is 50, key is test_row_0/C:col10/1731191987384/Put/seqid=0 2024-11-09T22:39:47,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741839_1015 (size=12001) 2024-11-09T22:39:47,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T22:39:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:47,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:47,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:47,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T22:39:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192047995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192047996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192047997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192047997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192047997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T22:39:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T22:39:48,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:48,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/4c23aa3ca0ba4734a308b18e802abac0 2024-11-09T22:39:48,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/e9e167697c7b424e930e1d48813999fa as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e9e167697c7b424e930e1d48813999fa 2024-11-09T22:39:48,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e9e167697c7b424e930e1d48813999fa, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T22:39:48,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/bdd93c98994a4a74867ad5a203c49fa3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bdd93c98994a4a74867ad5a203c49fa3 2024-11-09T22:39:48,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bdd93c98994a4a74867ad5a203c49fa3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T22:39:48,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/4c23aa3ca0ba4734a308b18e802abac0 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4c23aa3ca0ba4734a308b18e802abac0 2024-11-09T22:39:48,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192048306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192048307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192048307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4c23aa3ca0ba4734a308b18e802abac0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T22:39:48,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1f18320792a575fd182c9198939e0c16 in 875ms, sequenceid=14, compaction requested=false 2024-11-09T22:39:48,341 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T22:39:48,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:48,360 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:48,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-09T22:39:48,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T22:39:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/bcc626afa89d4b6f8ab27178596b4a06 is 50, key is test_row_0/A:col10/1731191988356/Put/seqid=0 2024-11-09T22:39:48,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192048393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192048397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741840_1016 (size=16681) 2024-11-09T22:39:48,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/bcc626afa89d4b6f8ab27178596b4a06 2024-11-09T22:39:48,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/b488e71f9fee4631a60e7477bd185b97 is 50, key is test_row_0/B:col10/1731191988356/Put/seqid=0 2024-11-09T22:39:48,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741841_1017 (size=12001) 2024-11-09T22:39:48,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/b488e71f9fee4631a60e7477bd185b97 2024-11-09T22:39:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T22:39:48,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192048506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192048507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/30aea86b93d242da8113e3635de206e1 is 50, key is test_row_0/C:col10/1731191988356/Put/seqid=0 2024-11-09T22:39:48,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T22:39:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:48,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:48,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741842_1018 (size=12001) 2024-11-09T22:39:48,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/30aea86b93d242da8113e3635de206e1 2024-11-09T22:39:48,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/bcc626afa89d4b6f8ab27178596b4a06 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bcc626afa89d4b6f8ab27178596b4a06 2024-11-09T22:39:48,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bcc626afa89d4b6f8ab27178596b4a06, entries=250, sequenceid=42, filesize=16.3 K 2024-11-09T22:39:48,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/b488e71f9fee4631a60e7477bd185b97 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/b488e71f9fee4631a60e7477bd185b97 2024-11-09T22:39:48,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/b488e71f9fee4631a60e7477bd185b97, entries=150, sequenceid=42, filesize=11.7 K 2024-11-09T22:39:48,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/30aea86b93d242da8113e3635de206e1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/30aea86b93d242da8113e3635de206e1 2024-11-09T22:39:48,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/30aea86b93d242da8113e3635de206e1, entries=150, sequenceid=42, filesize=11.7 K 2024-11-09T22:39:48,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 1f18320792a575fd182c9198939e0c16 in 307ms, sequenceid=42, compaction requested=false 2024-11-09T22:39:48,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:48,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T22:39:48,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:48,688 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T22:39:48,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/f7735160aae0429d966813e149a49a23 is 50, key is test_row_0/A:col10/1731191988393/Put/seqid=0 2024-11-09T22:39:48,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741843_1019 (size=9657) 2024-11-09T22:39:48,744 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/f7735160aae0429d966813e149a49a23 2024-11-09T22:39:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:48,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:48,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/5ce4d9f109d04758b7f17a8462db7a90 is 50, key is test_row_0/B:col10/1731191988393/Put/seqid=0 2024-11-09T22:39:48,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741844_1020 (size=9657) 2024-11-09T22:39:48,815 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/5ce4d9f109d04758b7f17a8462db7a90 2024-11-09T22:39:48,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/578629df320546139ea408ecb6625da9 is 50, key is test_row_0/C:col10/1731191988393/Put/seqid=0 2024-11-09T22:39:48,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741845_1021 (size=9657) 2024-11-09T22:39:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192048903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192048906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192048909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192048913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192048917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192049025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192049036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192049037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192049038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192049043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192049235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192049245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192049245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192049256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192049256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,275 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/578629df320546139ea408ecb6625da9 2024-11-09T22:39:49,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/f7735160aae0429d966813e149a49a23 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/f7735160aae0429d966813e149a49a23 2024-11-09T22:39:49,314 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/f7735160aae0429d966813e149a49a23, entries=100, sequenceid=50, filesize=9.4 K 2024-11-09T22:39:49,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/5ce4d9f109d04758b7f17a8462db7a90 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/5ce4d9f109d04758b7f17a8462db7a90 2024-11-09T22:39:49,333 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/5ce4d9f109d04758b7f17a8462db7a90, entries=100, sequenceid=50, filesize=9.4 K 2024-11-09T22:39:49,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/578629df320546139ea408ecb6625da9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/578629df320546139ea408ecb6625da9 2024-11-09T22:39:49,361 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/578629df320546139ea408ecb6625da9, entries=100, sequenceid=50, filesize=9.4 K 2024-11-09T22:39:49,364 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 1f18320792a575fd182c9198939e0c16 in 677ms, sequenceid=50, compaction requested=true 2024-11-09T22:39:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-09T22:39:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-09T22:39:49,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T22:39:49,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-11-09T22:39:49,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.0730 sec 2024-11-09T22:39:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T22:39:49,496 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:39:49,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:39:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-09T22:39:49,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:39:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T22:39:49,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:39:49,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:39:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:49,552 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:49,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-09T22:39:49,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/dea556156bca4db6a597f80c6d023cdd is 50, key is test_row_0/A:col10/1731191988900/Put/seqid=0 2024-11-09T22:39:49,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192049565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192049567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192049566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192049570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192049573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T22:39:49,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741846_1022 (size=14341) 2024-11-09T22:39:49,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/dea556156bca4db6a597f80c6d023cdd 2024-11-09T22:39:49,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/8a4a5499d6194c6fa10663604f873dba is 50, key is test_row_0/B:col10/1731191988900/Put/seqid=0 2024-11-09T22:39:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741847_1023 (size=12001) 2024-11-09T22:39:49,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/8a4a5499d6194c6fa10663604f873dba 2024-11-09T22:39:49,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T22:39:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:49,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:49,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192049685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192049686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192049686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:49,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/c9af2d14cc844f60a0a048b1b66bcea1 is 50, key is test_row_0/C:col10/1731191988900/Put/seqid=0 2024-11-09T22:39:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741848_1024 (size=12001) 2024-11-09T22:39:49,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/c9af2d14cc844f60a0a048b1b66bcea1 2024-11-09T22:39:49,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/dea556156bca4db6a597f80c6d023cdd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/dea556156bca4db6a597f80c6d023cdd 2024-11-09T22:39:49,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/dea556156bca4db6a597f80c6d023cdd, entries=200, sequenceid=79, filesize=14.0 K 2024-11-09T22:39:49,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/8a4a5499d6194c6fa10663604f873dba as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8a4a5499d6194c6fa10663604f873dba 2024-11-09T22:39:49,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8a4a5499d6194c6fa10663604f873dba, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T22:39:49,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/c9af2d14cc844f60a0a048b1b66bcea1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c9af2d14cc844f60a0a048b1b66bcea1 2024-11-09T22:39:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T22:39:49,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c9af2d14cc844f60a0a048b1b66bcea1, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T22:39:49,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 1f18320792a575fd182c9198939e0c16 in 266ms, sequenceid=79, compaction requested=true 2024-11-09T22:39:49,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:49,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T22:39:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:49,828 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T22:39:49,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:39:49,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:49,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:39:49,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:49,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:39:49,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:39:49,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:49,841 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:39:49,844 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:39:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/7bb1c56391064abe9a7c06489263a6d3 is 50, key is test_row_0/A:col10/1731191989559/Put/seqid=0 2024-11-09T22:39:49,848 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:39:49,848 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:39:49,848 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:39:49,849 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:49,849 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:49,849 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e9e167697c7b424e930e1d48813999fa, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bcc626afa89d4b6f8ab27178596b4a06, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/f7735160aae0429d966813e149a49a23, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/dea556156bca4db6a597f80c6d023cdd] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=51.4 K 2024-11-09T22:39:49,849 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bdd93c98994a4a74867ad5a203c49fa3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/b488e71f9fee4631a60e7477bd185b97, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/5ce4d9f109d04758b7f17a8462db7a90, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8a4a5499d6194c6fa10663604f873dba] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=44.6 K 2024-11-09T22:39:49,851 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9e167697c7b424e930e1d48813999fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731191987384 2024-11-09T22:39:49,852 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcc626afa89d4b6f8ab27178596b4a06, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731191987607 2024-11-09T22:39:49,854 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7735160aae0429d966813e149a49a23, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731191988379 2024-11-09T22:39:49,855 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting bdd93c98994a4a74867ad5a203c49fa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731191987384 2024-11-09T22:39:49,856 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting dea556156bca4db6a597f80c6d023cdd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731191988900 2024-11-09T22:39:49,862 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting b488e71f9fee4631a60e7477bd185b97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731191987624 2024-11-09T22:39:49,863 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ce4d9f109d04758b7f17a8462db7a90, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731191988379 2024-11-09T22:39:49,864 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a4a5499d6194c6fa10663604f873dba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731191988900 2024-11-09T22:39:49,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741849_1025 (size=12001) 2024-11-09T22:39:49,885 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/7bb1c56391064abe9a7c06489263a6d3 2024-11-09T22:39:49,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/80b3efe5424e42e79c89a63f19a253f8 is 50, key is test_row_0/B:col10/1731191989559/Put/seqid=0 2024-11-09T22:39:49,912 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#13 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:49,913 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/ffb15218afb643248da38279bae7030c is 50, key is test_row_0/B:col10/1731191988900/Put/seqid=0 2024-11-09T22:39:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:49,925 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#15 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:49,926 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/1bb4ebcc5d4d4130bf7eeca69fae1222 is 50, key is test_row_0/A:col10/1731191988900/Put/seqid=0 2024-11-09T22:39:49,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741851_1027 (size=12001) 2024-11-09T22:39:49,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741850_1026 (size=12139) 2024-11-09T22:39:49,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741852_1028 (size=12139) 2024-11-09T22:39:50,003 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/1bb4ebcc5d4d4130bf7eeca69fae1222 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1bb4ebcc5d4d4130bf7eeca69fae1222 2024-11-09T22:39:50,029 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into 1bb4ebcc5d4d4130bf7eeca69fae1222(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:50,029 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:50,031 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=12, startTime=1731191989819; duration=0sec 2024-11-09T22:39:50,032 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:50,032 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:39:50,032 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:39:50,036 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:39:50,037 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:39:50,037 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:50,037 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4c23aa3ca0ba4734a308b18e802abac0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/30aea86b93d242da8113e3635de206e1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/578629df320546139ea408ecb6625da9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c9af2d14cc844f60a0a048b1b66bcea1] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=44.6 K 2024-11-09T22:39:50,038 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c23aa3ca0ba4734a308b18e802abac0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731191987384 2024-11-09T22:39:50,043 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30aea86b93d242da8113e3635de206e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731191987624 2024-11-09T22:39:50,051 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T22:39:50,051 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 578629df320546139ea408ecb6625da9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731191988379 2024-11-09T22:39:50,056 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9af2d14cc844f60a0a048b1b66bcea1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731191988900 2024-11-09T22:39:50,094 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#16 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:50,094 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/90bf07e5e5d34a2b957f0ccf67991cd1 is 50, key is test_row_0/C:col10/1731191988900/Put/seqid=0 2024-11-09T22:39:50,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741853_1029 (size=12139) 2024-11-09T22:39:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T22:39:50,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192050114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192050120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,152 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T22:39:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192050124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192050140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192050146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,159 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/90bf07e5e5d34a2b957f0ccf67991cd1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/90bf07e5e5d34a2b957f0ccf67991cd1 2024-11-09T22:39:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,177 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into 90bf07e5e5d34a2b957f0ccf67991cd1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:50,177 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:50,178 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=12, startTime=1731191989837; duration=0sec 2024-11-09T22:39:50,178 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:50,178 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:39:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192050255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192050255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192050265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192050265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192050268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,368 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/80b3efe5424e42e79c89a63f19a253f8 2024-11-09T22:39:50,415 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/ffb15218afb643248da38279bae7030c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ffb15218afb643248da38279bae7030c 2024-11-09T22:39:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/d5363b2f9ccb4e7a86d161556562b6d5 is 50, key is test_row_0/C:col10/1731191989559/Put/seqid=0 2024-11-09T22:39:50,442 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into ffb15218afb643248da38279bae7030c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:50,442 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:50,442 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=12, startTime=1731191989834; duration=0sec 2024-11-09T22:39:50,443 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:50,443 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:39:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741854_1030 (size=12001) 2024-11-09T22:39:50,447 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/d5363b2f9ccb4e7a86d161556562b6d5 2024-11-09T22:39:50,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/7bb1c56391064abe9a7c06489263a6d3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7bb1c56391064abe9a7c06489263a6d3 2024-11-09T22:39:50,474 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7bb1c56391064abe9a7c06489263a6d3, entries=150, sequenceid=86, filesize=11.7 K 2024-11-09T22:39:50,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/80b3efe5424e42e79c89a63f19a253f8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/80b3efe5424e42e79c89a63f19a253f8 2024-11-09T22:39:50,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192050477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192050477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192050477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,491 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/80b3efe5424e42e79c89a63f19a253f8, entries=150, sequenceid=86, filesize=11.7 K 2024-11-09T22:39:50,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192050477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192050477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/d5363b2f9ccb4e7a86d161556562b6d5 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/d5363b2f9ccb4e7a86d161556562b6d5 2024-11-09T22:39:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,507 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/d5363b2f9ccb4e7a86d161556562b6d5, entries=150, sequenceid=86, filesize=11.7 K 2024-11-09T22:39:50,511 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 1f18320792a575fd182c9198939e0c16 in 682ms, sequenceid=86, compaction requested=false 2024-11-09T22:39:50,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:50,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:50,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-09T22:39:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-09T22:39:50,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T22:39:50,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-09T22:39:50,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0280 sec 2024-11-09T22:39:50,531 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T22:39:50,531 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T22:39:50,534 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T22:39:50,534 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T22:39:50,534 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T22:39:50,534 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T22:39:50,536 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T22:39:50,536 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T22:39:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T22:39:50,636 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:39:50,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:39:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-09T22:39:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T22:39:50,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:39:50,642 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:39:50,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:39:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T22:39:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:50,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:50,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T22:39:50,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-09T22:39:50,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:50,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:50,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:50,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:50,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:50,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192050805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192050805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/2b77b618f1a24065bbdf6abc8f7a206f is 50, key is test_row_0/A:col10/1731191990798/Put/seqid=0 2024-11-09T22:39:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192050805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192050822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192050823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741855_1031 (size=16681) 2024-11-09T22:39:50,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/2b77b618f1a24065bbdf6abc8f7a206f 2024-11-09T22:39:50,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/018f624986e8422888f37cae713a1ef8 is 50, key is test_row_0/B:col10/1731191990798/Put/seqid=0 2024-11-09T22:39:50,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192050931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192050936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192050936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T22:39:50,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T22:39:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:50,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741856_1032 (size=12001) 2024-11-09T22:39:51,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T22:39:51,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:51,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192051145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192051145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192051145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T22:39:51,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T22:39:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192051325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:51,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192051326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:51,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/018f624986e8422888f37cae713a1ef8 2024-11-09T22:39:51,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/cafd707a6a2e488da0f4e2b222f8a244 is 50, key is test_row_0/C:col10/1731191990798/Put/seqid=0 2024-11-09T22:39:51,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T22:39:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:51,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:51,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741857_1033 (size=12001) 2024-11-09T22:39:51,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/cafd707a6a2e488da0f4e2b222f8a244 2024-11-09T22:39:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/2b77b618f1a24065bbdf6abc8f7a206f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/2b77b618f1a24065bbdf6abc8f7a206f 2024-11-09T22:39:51,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192051465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:51,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192051465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:51,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192051465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:51,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/2b77b618f1a24065bbdf6abc8f7a206f, entries=250, sequenceid=121, filesize=16.3 K 2024-11-09T22:39:51,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/018f624986e8422888f37cae713a1ef8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/018f624986e8422888f37cae713a1ef8 2024-11-09T22:39:51,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/018f624986e8422888f37cae713a1ef8, entries=150, sequenceid=121, filesize=11.7 K 2024-11-09T22:39:51,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/cafd707a6a2e488da0f4e2b222f8a244 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cafd707a6a2e488da0f4e2b222f8a244 2024-11-09T22:39:51,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cafd707a6a2e488da0f4e2b222f8a244, entries=150, sequenceid=121, filesize=11.7 K 2024-11-09T22:39:51,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for 1f18320792a575fd182c9198939e0c16 in 735ms, sequenceid=121, compaction requested=true 2024-11-09T22:39:51,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:51,537 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:51,540 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:51,541 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:39:51,541 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,541 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1bb4ebcc5d4d4130bf7eeca69fae1222, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7bb1c56391064abe9a7c06489263a6d3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/2b77b618f1a24065bbdf6abc8f7a206f] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=39.9 K 2024-11-09T22:39:51,544 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bb4ebcc5d4d4130bf7eeca69fae1222, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731191988900 2024-11-09T22:39:51,545 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb1c56391064abe9a7c06489263a6d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731191989559 2024-11-09T22:39:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:39:51,547 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b77b618f1a24065bbdf6abc8f7a206f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731191990120 2024-11-09T22:39:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:39:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:51,547 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:39:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:51,553 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:51,553 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:39:51,553 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,554 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ffb15218afb643248da38279bae7030c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/80b3efe5424e42e79c89a63f19a253f8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/018f624986e8422888f37cae713a1ef8] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=35.3 K 2024-11-09T22:39:51,555 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ffb15218afb643248da38279bae7030c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731191988900 2024-11-09T22:39:51,557 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 80b3efe5424e42e79c89a63f19a253f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731191989559 2024-11-09T22:39:51,558 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 018f624986e8422888f37cae713a1ef8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731191990120 2024-11-09T22:39:51,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T22:39:51,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,593 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-09T22:39:51,601 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:51,602 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/1c0db947758f4b9f9c056b7c5206b161 is 50, key is test_row_0/B:col10/1731191990798/Put/seqid=0 2024-11-09T22:39:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4049b983abba4541849d882fe344ee92 is 50, key is test_row_0/A:col10/1731191990812/Put/seqid=0 2024-11-09T22:39:51,617 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:51,618 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/1ac0311c59f2484ea2b88fe1a6611088 is 50, key is test_row_0/A:col10/1731191990798/Put/seqid=0 2024-11-09T22:39:51,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741859_1035 (size=9657) 2024-11-09T22:39:51,668 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4049b983abba4541849d882fe344ee92 2024-11-09T22:39:51,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741858_1034 (size=12241) 2024-11-09T22:39:51,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741860_1036 (size=12241) 2024-11-09T22:39:51,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/9a7be23ec5f34c9d9103ea6e7f282101 is 50, key is test_row_0/B:col10/1731191990812/Put/seqid=0 2024-11-09T22:39:51,701 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/1ac0311c59f2484ea2b88fe1a6611088 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1ac0311c59f2484ea2b88fe1a6611088 2024-11-09T22:39:51,701 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/1c0db947758f4b9f9c056b7c5206b161 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/1c0db947758f4b9f9c056b7c5206b161 2024-11-09T22:39:51,724 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into 1c0db947758f4b9f9c056b7c5206b161(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:51,724 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:51,724 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=13, startTime=1731191991547; duration=0sec 2024-11-09T22:39:51,725 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:51,725 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:39:51,725 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:51,727 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into 1ac0311c59f2484ea2b88fe1a6611088(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:51,727 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:51,727 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=13, startTime=1731191991537; duration=0sec 2024-11-09T22:39:51,728 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:51,728 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:39:51,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741861_1037 (size=9657) 2024-11-09T22:39:51,731 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/9a7be23ec5f34c9d9103ea6e7f282101 2024-11-09T22:39:51,732 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:51,732 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:39:51,732 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,733 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/90bf07e5e5d34a2b957f0ccf67991cd1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/d5363b2f9ccb4e7a86d161556562b6d5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cafd707a6a2e488da0f4e2b222f8a244] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=35.3 K 2024-11-09T22:39:51,733 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 90bf07e5e5d34a2b957f0ccf67991cd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731191988900 2024-11-09T22:39:51,735 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d5363b2f9ccb4e7a86d161556562b6d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731191989559 2024-11-09T22:39:51,737 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting cafd707a6a2e488da0f4e2b222f8a244, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731191990120 2024-11-09T22:39:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/ed09de8316eb4dcc819afa839ac4f567 is 50, key is test_row_0/C:col10/1731191990812/Put/seqid=0 2024-11-09T22:39:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T22:39:51,782 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#26 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:51,783 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/20fa1e3f6ca048f291d1c343ef8e00be is 50, key is test_row_0/C:col10/1731191990798/Put/seqid=0 2024-11-09T22:39:51,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741862_1038 (size=9657) 2024-11-09T22:39:51,805 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/ed09de8316eb4dcc819afa839ac4f567 2024-11-09T22:39:51,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4049b983abba4541849d882fe344ee92 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4049b983abba4541849d882fe344ee92 2024-11-09T22:39:51,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741863_1039 (size=12241) 2024-11-09T22:39:51,831 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4049b983abba4541849d882fe344ee92, entries=100, sequenceid=126, filesize=9.4 K 2024-11-09T22:39:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/9a7be23ec5f34c9d9103ea6e7f282101 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9a7be23ec5f34c9d9103ea6e7f282101 2024-11-09T22:39:51,841 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/20fa1e3f6ca048f291d1c343ef8e00be as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/20fa1e3f6ca048f291d1c343ef8e00be 2024-11-09T22:39:51,851 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9a7be23ec5f34c9d9103ea6e7f282101, entries=100, sequenceid=126, filesize=9.4 K 2024-11-09T22:39:51,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/ed09de8316eb4dcc819afa839ac4f567 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/ed09de8316eb4dcc819afa839ac4f567 2024-11-09T22:39:51,862 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into 20fa1e3f6ca048f291d1c343ef8e00be(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:51,863 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:51,863 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=13, startTime=1731191991547; duration=0sec 2024-11-09T22:39:51,863 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:51,863 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:39:51,901 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/ed09de8316eb4dcc819afa839ac4f567, entries=100, sequenceid=126, filesize=9.4 K 2024-11-09T22:39:51,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1f18320792a575fd182c9198939e0c16 in 311ms, sequenceid=126, compaction requested=false 2024-11-09T22:39:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-09T22:39:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-09T22:39:51,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-09T22:39:51,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2650 sec 2024-11-09T22:39:51,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2760 sec 2024-11-09T22:39:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:52,014 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:52,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:39:52,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/9066252f5a56456a82a2079ac6dde416 is 50, key is test_row_0/A:col10/1731191992006/Put/seqid=0 2024-11-09T22:39:52,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741864_1040 (size=14541) 2024-11-09T22:39:52,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/9066252f5a56456a82a2079ac6dde416 2024-11-09T22:39:52,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/f46ccec201a54a3d85c3585cf04555d6 is 50, key is test_row_0/B:col10/1731191992006/Put/seqid=0 2024-11-09T22:39:52,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192052135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192052140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192052138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741865_1041 (size=12151) 2024-11-09T22:39:52,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192052256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192052256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192052256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192052355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192052355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,363 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 1f18320792a575fd182c9198939e0c16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:39:52,363 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1f18320792a575fd182c9198939e0c16 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:39:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192052465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192052467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192052471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/f46ccec201a54a3d85c3585cf04555d6 2024-11-09T22:39:52,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/394b245be35b456a827548d867eccb6e is 50, key is test_row_0/C:col10/1731191992006/Put/seqid=0 2024-11-09T22:39:52,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741866_1042 (size=12151) 2024-11-09T22:39:52,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/394b245be35b456a827548d867eccb6e 2024-11-09T22:39:52,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/9066252f5a56456a82a2079ac6dde416 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/9066252f5a56456a82a2079ac6dde416 2024-11-09T22:39:52,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/9066252f5a56456a82a2079ac6dde416, entries=200, sequenceid=140, filesize=14.2 K 2024-11-09T22:39:52,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/f46ccec201a54a3d85c3585cf04555d6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/f46ccec201a54a3d85c3585cf04555d6 2024-11-09T22:39:52,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/f46ccec201a54a3d85c3585cf04555d6, entries=150, sequenceid=140, filesize=11.9 K 2024-11-09T22:39:52,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/394b245be35b456a827548d867eccb6e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/394b245be35b456a827548d867eccb6e 2024-11-09T22:39:52,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/394b245be35b456a827548d867eccb6e, entries=150, sequenceid=140, filesize=11.9 K 2024-11-09T22:39:52,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1f18320792a575fd182c9198939e0c16 in 743ms, sequenceid=140, compaction requested=true 2024-11-09T22:39:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:52,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:39:52,758 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:52,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:52,758 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:52,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:39:52,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:52,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:39:52,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:52,760 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:52,760 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:39:52,760 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:52,760 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1ac0311c59f2484ea2b88fe1a6611088, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4049b983abba4541849d882fe344ee92, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/9066252f5a56456a82a2079ac6dde416] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=35.6 K 2024-11-09T22:39:52,760 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:52,760 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:39:52,761 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:52,761 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/1c0db947758f4b9f9c056b7c5206b161, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9a7be23ec5f34c9d9103ea6e7f282101, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/f46ccec201a54a3d85c3585cf04555d6] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=33.3 K 2024-11-09T22:39:52,761 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ac0311c59f2484ea2b88fe1a6611088, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731191990120 2024-11-09T22:39:52,761 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0db947758f4b9f9c056b7c5206b161, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731191990120 2024-11-09T22:39:52,762 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a7be23ec5f34c9d9103ea6e7f282101, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731191990809 2024-11-09T22:39:52,762 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4049b983abba4541849d882fe344ee92, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731191990809 2024-11-09T22:39:52,763 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting f46ccec201a54a3d85c3585cf04555d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731191992006 2024-11-09T22:39:52,763 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9066252f5a56456a82a2079ac6dde416, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731191992003 2024-11-09T22:39:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T22:39:52,786 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:39:52,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:39:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-09T22:39:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:52,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T22:39:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T22:39:52,794 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:39:52,796 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:39:52,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:39:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:52,805 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:52,806 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/c5d1203ade834f7c8a6293167a3ac983 is 50, key is test_row_0/A:col10/1731191992006/Put/seqid=0 2024-11-09T22:39:52,813 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:52,813 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/0589debe75004d9ea1001118388e0491 is 50, key is test_row_0/B:col10/1731191992006/Put/seqid=0 2024-11-09T22:39:52,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/a1bb68767ba346699dbbbb3d08dd5dfe is 50, key is test_row_0/A:col10/1731191992129/Put/seqid=0 2024-11-09T22:39:52,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192052841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192052842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192052844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741868_1044 (size=12493) 2024-11-09T22:39:52,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741867_1043 (size=12493) 2024-11-09T22:39:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T22:39:52,906 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/0589debe75004d9ea1001118388e0491 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/0589debe75004d9ea1001118388e0491 2024-11-09T22:39:52,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741869_1045 (size=12151) 2024-11-09T22:39:52,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/a1bb68767ba346699dbbbb3d08dd5dfe 2024-11-09T22:39:52,923 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into 0589debe75004d9ea1001118388e0491(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:52,924 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:52,924 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=13, startTime=1731191992758; duration=0sec 2024-11-09T22:39:52,924 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:52,924 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:39:52,924 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:52,926 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:52,926 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:39:52,927 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:52,927 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/20fa1e3f6ca048f291d1c343ef8e00be, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/ed09de8316eb4dcc819afa839ac4f567, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/394b245be35b456a827548d867eccb6e] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=33.3 K 2024-11-09T22:39:52,929 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 20fa1e3f6ca048f291d1c343ef8e00be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731191990120 2024-11-09T22:39:52,930 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ed09de8316eb4dcc819afa839ac4f567, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731191990809 2024-11-09T22:39:52,930 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 394b245be35b456a827548d867eccb6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731191992006 2024-11-09T22:39:52,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4bb6d0585aad40b6a5f739b4668dfaae is 50, key is test_row_0/B:col10/1731191992129/Put/seqid=0 2024-11-09T22:39:52,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T22:39:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:52,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:52,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192052955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192052955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192052965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:52,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741870_1046 (size=12151) 2024-11-09T22:39:52,999 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:53,000 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/01ceb183123142a2acf84b72cce90b44 is 50, key is test_row_0/C:col10/1731191992006/Put/seqid=0 2024-11-09T22:39:53,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4bb6d0585aad40b6a5f739b4668dfaae 2024-11-09T22:39:53,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741871_1047 (size=12493) 2024-11-09T22:39:53,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/de54bc44d2fc4f1ba92f28310380d394 is 50, key is test_row_0/C:col10/1731191992129/Put/seqid=0 2024-11-09T22:39:53,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T22:39:53,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:53,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741872_1048 (size=12151) 2024-11-09T22:39:53,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/de54bc44d2fc4f1ba92f28310380d394 2024-11-09T22:39:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T22:39:53,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/a1bb68767ba346699dbbbb3d08dd5dfe as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/a1bb68767ba346699dbbbb3d08dd5dfe 2024-11-09T22:39:53,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/a1bb68767ba346699dbbbb3d08dd5dfe, entries=150, sequenceid=167, filesize=11.9 K 2024-11-09T22:39:53,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4bb6d0585aad40b6a5f739b4668dfaae as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4bb6d0585aad40b6a5f739b4668dfaae 2024-11-09T22:39:53,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4bb6d0585aad40b6a5f739b4668dfaae, entries=150, sequenceid=167, filesize=11.9 K 2024-11-09T22:39:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/de54bc44d2fc4f1ba92f28310380d394 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/de54bc44d2fc4f1ba92f28310380d394 2024-11-09T22:39:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192053165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:53,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/de54bc44d2fc4f1ba92f28310380d394, entries=150, sequenceid=167, filesize=11.9 K 2024-11-09T22:39:53,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 1f18320792a575fd182c9198939e0c16 in 389ms, sequenceid=167, compaction requested=false 2024-11-09T22:39:53,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:53,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:39:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:53,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4ef6297fe66d4b8b86f32586388627de is 50, key is test_row_0/A:col10/1731191992840/Put/seqid=0 2024-11-09T22:39:53,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741873_1049 (size=14541) 2024-11-09T22:39:53,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T22:39:53,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:53,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,314 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/c5d1203ade834f7c8a6293167a3ac983 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/c5d1203ade834f7c8a6293167a3ac983 2024-11-09T22:39:53,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192053308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192053311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:53,325 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into c5d1203ade834f7c8a6293167a3ac983(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:53,325 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:53,325 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=13, startTime=1731191992757; duration=0sec 2024-11-09T22:39:53,326 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:53,326 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:39:53,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T22:39:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T22:39:53,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192053426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192053435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:53,478 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/01ceb183123142a2acf84b72cce90b44 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/01ceb183123142a2acf84b72cce90b44 2024-11-09T22:39:53,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192053485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:53,489 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into 01ceb183123142a2acf84b72cce90b44(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:53,489 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:53,489 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=13, startTime=1731191992758; duration=0sec 2024-11-09T22:39:53,489 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:53,489 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:39:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:53,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T22:39:53,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4ef6297fe66d4b8b86f32586388627de 2024-11-09T22:39:53,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192053635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:53,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192053646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:53,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/ca81ee443eeb4e849235b0691e85272d is 50, key is test_row_0/B:col10/1731191992840/Put/seqid=0 2024-11-09T22:39:53,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741874_1050 (size=12151) 2024-11-09T22:39:53,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/ca81ee443eeb4e849235b0691e85272d 2024-11-09T22:39:53,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/9b0293f4617f4ec3a71f0298edc8eb16 is 50, key is test_row_0/C:col10/1731191992840/Put/seqid=0 2024-11-09T22:39:53,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T22:39:53,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:53,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741875_1051 (size=12151) 2024-11-09T22:39:53,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/9b0293f4617f4ec3a71f0298edc8eb16 2024-11-09T22:39:53,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4ef6297fe66d4b8b86f32586388627de as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4ef6297fe66d4b8b86f32586388627de 2024-11-09T22:39:53,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4ef6297fe66d4b8b86f32586388627de, entries=200, sequenceid=179, filesize=14.2 K 2024-11-09T22:39:53,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/ca81ee443eeb4e849235b0691e85272d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ca81ee443eeb4e849235b0691e85272d 2024-11-09T22:39:53,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ca81ee443eeb4e849235b0691e85272d, entries=150, sequenceid=179, filesize=11.9 K 2024-11-09T22:39:53,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/9b0293f4617f4ec3a71f0298edc8eb16 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/9b0293f4617f4ec3a71f0298edc8eb16 2024-11-09T22:39:53,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/9b0293f4617f4ec3a71f0298edc8eb16, entries=150, sequenceid=179, filesize=11.9 K 2024-11-09T22:39:53,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1f18320792a575fd182c9198939e0c16 in 643ms, sequenceid=179, compaction requested=true 2024-11-09T22:39:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:53,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:39:53,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:53,830 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:53,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:39:53,830 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:53,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:53,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:39:53,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:53,831 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:53,832 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:39:53,832 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,832 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/c5d1203ade834f7c8a6293167a3ac983, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/a1bb68767ba346699dbbbb3d08dd5dfe, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4ef6297fe66d4b8b86f32586388627de] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=38.3 K 2024-11-09T22:39:53,832 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:53,832 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:39:53,832 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,833 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/0589debe75004d9ea1001118388e0491, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4bb6d0585aad40b6a5f739b4668dfaae, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ca81ee443eeb4e849235b0691e85272d] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=35.9 K 2024-11-09T22:39:53,833 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5d1203ade834f7c8a6293167a3ac983, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731191992006 2024-11-09T22:39:53,833 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1bb68767ba346699dbbbb3d08dd5dfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731191992129 2024-11-09T22:39:53,833 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0589debe75004d9ea1001118388e0491, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731191992006 2024-11-09T22:39:53,834 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef6297fe66d4b8b86f32586388627de, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731191992826 2024-11-09T22:39:53,834 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb6d0585aad40b6a5f739b4668dfaae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731191992129 2024-11-09T22:39:53,835 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ca81ee443eeb4e849235b0691e85272d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731191992839 2024-11-09T22:39:53,853 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:53,860 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/073194f889dd46eabdc979b4eb3597ff is 50, key is test_row_0/A:col10/1731191992840/Put/seqid=0 2024-11-09T22:39:53,861 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:53,862 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/cd54e7a96f844bd6a1db7af8c9f01dc4 is 50, key is test_row_0/B:col10/1731191992840/Put/seqid=0 2024-11-09T22:39:53,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T22:39:53,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,892 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:39:53,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741876_1052 (size=12595) 2024-11-09T22:39:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/6103b0172dc4470f973f529c76fda9f3 is 50, key is test_row_0/A:col10/1731191993296/Put/seqid=0 2024-11-09T22:39:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T22:39:53,937 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/073194f889dd46eabdc979b4eb3597ff as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/073194f889dd46eabdc979b4eb3597ff 2024-11-09T22:39:53,949 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into 073194f889dd46eabdc979b4eb3597ff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:53,949 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:53,949 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=13, startTime=1731191993829; duration=0sec 2024-11-09T22:39:53,949 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:53,949 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:39:53,950 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:53,952 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:53,952 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:39:53,952 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:53,952 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/01ceb183123142a2acf84b72cce90b44, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/de54bc44d2fc4f1ba92f28310380d394, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/9b0293f4617f4ec3a71f0298edc8eb16] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=35.9 K 2024-11-09T22:39:53,953 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01ceb183123142a2acf84b72cce90b44, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731191992006 2024-11-09T22:39:53,954 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting de54bc44d2fc4f1ba92f28310380d394, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731191992129 2024-11-09T22:39:53,955 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b0293f4617f4ec3a71f0298edc8eb16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731191992839 2024-11-09T22:39:53,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741877_1053 (size=12595) 2024-11-09T22:39:53,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741878_1054 (size=12151) 2024-11-09T22:39:53,979 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:53,980 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/c0c2ccd1d3d64d06a7be4e48d985cfdc is 50, key is test_row_0/C:col10/1731191992840/Put/seqid=0 2024-11-09T22:39:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:54,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:54,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741879_1055 (size=12595) 2024-11-09T22:39:54,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192054033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,042 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/c0c2ccd1d3d64d06a7be4e48d985cfdc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c0c2ccd1d3d64d06a7be4e48d985cfdc 2024-11-09T22:39:54,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192054035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192054036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,055 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into c0c2ccd1d3d64d06a7be4e48d985cfdc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:54,055 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:54,055 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=13, startTime=1731191993830; duration=0sec 2024-11-09T22:39:54,055 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:54,055 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:39:54,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192054145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192054155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192054155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192054355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192054365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192054365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,378 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/6103b0172dc4470f973f529c76fda9f3 2024-11-09T22:39:54,379 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/cd54e7a96f844bd6a1db7af8c9f01dc4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/cd54e7a96f844bd6a1db7af8c9f01dc4 2024-11-09T22:39:54,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192054375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,384 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1f18320792a575fd182c9198939e0c16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:39:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,397 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into cd54e7a96f844bd6a1db7af8c9f01dc4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:54,397 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:54,397 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=13, startTime=1731191993830; duration=0sec 2024-11-09T22:39:54,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192054396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,399 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 1f18320792a575fd182c9198939e0c16 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:39:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,401 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:54,401 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:39:54,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e09a9b2f0f25449f8cbbf9d80c58b6b2 is 50, key is test_row_0/B:col10/1731191993296/Put/seqid=0 2024-11-09T22:39:54,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741880_1056 (size=12151) 2024-11-09T22:39:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192054665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192054686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192054686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:54,853 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e09a9b2f0f25449f8cbbf9d80c58b6b2 2024-11-09T22:39:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/f9a4b64aa163403ea24db4aa62abf6c2 is 50, key is test_row_0/C:col10/1731191993296/Put/seqid=0 2024-11-09T22:39:54,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741881_1057 (size=12151) 2024-11-09T22:39:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T22:39:55,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192055175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:55,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:55,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192055197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192055196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:55,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:55,315 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/f9a4b64aa163403ea24db4aa62abf6c2 2024-11-09T22:39:55,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/6103b0172dc4470f973f529c76fda9f3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6103b0172dc4470f973f529c76fda9f3 2024-11-09T22:39:55,346 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6103b0172dc4470f973f529c76fda9f3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-09T22:39:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e09a9b2f0f25449f8cbbf9d80c58b6b2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e09a9b2f0f25449f8cbbf9d80c58b6b2 2024-11-09T22:39:55,360 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e09a9b2f0f25449f8cbbf9d80c58b6b2, entries=150, sequenceid=206, filesize=11.9 K 2024-11-09T22:39:55,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/f9a4b64aa163403ea24db4aa62abf6c2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/f9a4b64aa163403ea24db4aa62abf6c2 2024-11-09T22:39:55,388 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/f9a4b64aa163403ea24db4aa62abf6c2, entries=150, sequenceid=206, filesize=11.9 K 2024-11-09T22:39:55,390 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 1f18320792a575fd182c9198939e0c16 in 1498ms, sequenceid=206, compaction requested=false 2024-11-09T22:39:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:55,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-09T22:39:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-09T22:39:55,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T22:39:55,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5970 sec 2024-11-09T22:39:55,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.6140 sec 2024-11-09T22:39:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:56,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:56,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T22:39:56,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/6179c18658a34d759d9101e00293d97d is 50, key is test_row_0/A:col10/1731191996196/Put/seqid=0 2024-11-09T22:39:56,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741882_1058 (size=12151) 2024-11-09T22:39:56,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/6179c18658a34d759d9101e00293d97d 2024-11-09T22:39:56,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/ab9caf0bf062491e87b3184fc3ff2211 is 50, key is test_row_0/B:col10/1731191996196/Put/seqid=0 2024-11-09T22:39:56,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741883_1059 (size=12151) 2024-11-09T22:39:56,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192056315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192056315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192056315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192056425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192056425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192056425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192056635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192056635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192056635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:56,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/ab9caf0bf062491e87b3184fc3ff2211 2024-11-09T22:39:56,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/cf362435bdd04b1788508e4424bcc302 is 50, key is test_row_0/C:col10/1731191996196/Put/seqid=0 2024-11-09T22:39:56,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741884_1060 (size=12151) 2024-11-09T22:39:56,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/cf362435bdd04b1788508e4424bcc302 2024-11-09T22:39:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/6179c18658a34d759d9101e00293d97d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6179c18658a34d759d9101e00293d97d 2024-11-09T22:39:56,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6179c18658a34d759d9101e00293d97d, entries=150, sequenceid=223, filesize=11.9 K 2024-11-09T22:39:56,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/ab9caf0bf062491e87b3184fc3ff2211 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ab9caf0bf062491e87b3184fc3ff2211 2024-11-09T22:39:56,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ab9caf0bf062491e87b3184fc3ff2211, entries=150, sequenceid=223, filesize=11.9 K 2024-11-09T22:39:56,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/cf362435bdd04b1788508e4424bcc302 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cf362435bdd04b1788508e4424bcc302 2024-11-09T22:39:56,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cf362435bdd04b1788508e4424bcc302, entries=150, sequenceid=223, filesize=11.9 K 2024-11-09T22:39:56,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 1f18320792a575fd182c9198939e0c16 in 631ms, sequenceid=223, compaction requested=true 2024-11-09T22:39:56,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:56,832 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:56,834 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:56,834 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:39:56,834 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:56,834 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/073194f889dd46eabdc979b4eb3597ff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6103b0172dc4470f973f529c76fda9f3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6179c18658a34d759d9101e00293d97d] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=36.0 K 2024-11-09T22:39:56,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:39:56,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:56,835 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:56,835 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 073194f889dd46eabdc979b4eb3597ff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731191992839 2024-11-09T22:39:56,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:39:56,836 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6103b0172dc4470f973f529c76fda9f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731191993281 2024-11-09T22:39:56,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:56,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:39:56,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:56,836 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:56,836 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:39:56,836 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:56,837 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/cd54e7a96f844bd6a1db7af8c9f01dc4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e09a9b2f0f25449f8cbbf9d80c58b6b2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ab9caf0bf062491e87b3184fc3ff2211] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=36.0 K 2024-11-09T22:39:56,837 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6179c18658a34d759d9101e00293d97d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731191994013 2024-11-09T22:39:56,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting cd54e7a96f844bd6a1db7af8c9f01dc4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731191992839 2024-11-09T22:39:56,838 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting e09a9b2f0f25449f8cbbf9d80c58b6b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731191993281 2024-11-09T22:39:56,839 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ab9caf0bf062491e87b3184fc3ff2211, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731191994013 2024-11-09T22:39:56,863 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:56,864 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/8c948f48318f496289a0a1c3a2bc056a is 50, key is test_row_0/A:col10/1731191996196/Put/seqid=0 2024-11-09T22:39:56,872 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:56,873 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/9801ca83991b49a79c1c5c86808a9068 is 50, key is test_row_0/B:col10/1731191996196/Put/seqid=0 2024-11-09T22:39:56,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741885_1061 (size=12697) 2024-11-09T22:39:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:56,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:39:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T22:39:56,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741886_1062 (size=12697) 2024-11-09T22:39:56,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:39:56,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:39:56,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/93d23f31e1b6408581054dd26f4ac977 is 50, key is test_row_0/A:col10/1731191996274/Put/seqid=0 2024-11-09T22:39:56,968 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/9801ca83991b49a79c1c5c86808a9068 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9801ca83991b49a79c1c5c86808a9068 2024-11-09T22:39:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-09T22:39:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T22:39:56,974 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:39:56,976 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:39:56,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:39:56,988 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into 9801ca83991b49a79c1c5c86808a9068(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:56,989 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:56,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741887_1063 (size=16931) 2024-11-09T22:39:56,989 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=13, startTime=1731191996835; duration=0sec 2024-11-09T22:39:56,989 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:56,989 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:39:56,989 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:39:56,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/93d23f31e1b6408581054dd26f4ac977 2024-11-09T22:39:56,991 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:39:56,991 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:39:56,991 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:56,992 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c0c2ccd1d3d64d06a7be4e48d985cfdc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/f9a4b64aa163403ea24db4aa62abf6c2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cf362435bdd04b1788508e4424bcc302] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=36.0 K 2024-11-09T22:39:56,995 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c2ccd1d3d64d06a7be4e48d985cfdc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731191992839 2024-11-09T22:39:56,997 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting f9a4b64aa163403ea24db4aa62abf6c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731191993281 2024-11-09T22:39:56,997 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting cf362435bdd04b1788508e4424bcc302, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731191994013 2024-11-09T22:39:57,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192056986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192057000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192057000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/8e83423c8643458f98831b0c36635012 is 50, key is test_row_0/B:col10/1731191996274/Put/seqid=0 2024-11-09T22:39:57,052 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:57,053 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/99fb485527014dc099c86b609b70ebdc is 50, key is test_row_0/C:col10/1731191996196/Put/seqid=0 2024-11-09T22:39:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T22:39:57,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741888_1064 (size=12151) 2024-11-09T22:39:57,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/8e83423c8643458f98831b0c36635012 2024-11-09T22:39:57,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/e28a1fb7dbc542529e59e711ddf7c62d is 50, key is test_row_0/C:col10/1731191996274/Put/seqid=0 2024-11-09T22:39:57,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741889_1065 (size=12697) 2024-11-09T22:39:57,115 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/99fb485527014dc099c86b609b70ebdc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/99fb485527014dc099c86b609b70ebdc 2024-11-09T22:39:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192057115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192057115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192057115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,127 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into 99fb485527014dc099c86b609b70ebdc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:57,128 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:57,128 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=13, startTime=1731191996836; duration=0sec 2024-11-09T22:39:57,128 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:57,128 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:39:57,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741890_1066 (size=12151) 2024-11-09T22:39:57,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T22:39:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:57,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/e28a1fb7dbc542529e59e711ddf7c62d 2024-11-09T22:39:57,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/93d23f31e1b6408581054dd26f4ac977 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/93d23f31e1b6408581054dd26f4ac977 2024-11-09T22:39:57,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/93d23f31e1b6408581054dd26f4ac977, entries=250, sequenceid=248, filesize=16.5 K 2024-11-09T22:39:57,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/8e83423c8643458f98831b0c36635012 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8e83423c8643458f98831b0c36635012 2024-11-09T22:39:57,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8e83423c8643458f98831b0c36635012, entries=150, sequenceid=248, filesize=11.9 K 2024-11-09T22:39:57,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/e28a1fb7dbc542529e59e711ddf7c62d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/e28a1fb7dbc542529e59e711ddf7c62d 2024-11-09T22:39:57,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/e28a1fb7dbc542529e59e711ddf7c62d, entries=150, sequenceid=248, filesize=11.9 K 2024-11-09T22:39:57,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1f18320792a575fd182c9198939e0c16 in 247ms, sequenceid=248, compaction requested=false 2024-11-09T22:39:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T22:39:57,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T22:39:57,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:57,288 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:39:57,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/83856ef206f4421b9a5ee6824c7663af is 50, key is test_row_0/A:col10/1731191996983/Put/seqid=0 2024-11-09T22:39:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:57,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741891_1067 (size=12251) 2024-11-09T22:39:57,362 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/8c948f48318f496289a0a1c3a2bc056a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8c948f48318f496289a0a1c3a2bc056a 2024-11-09T22:39:57,363 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/83856ef206f4421b9a5ee6824c7663af 2024-11-09T22:39:57,385 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into 8c948f48318f496289a0a1c3a2bc056a(size=12.4 K), total size for store is 28.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:57,385 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:57,385 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=13, startTime=1731191996832; duration=0sec 2024-11-09T22:39:57,386 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:57,386 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:39:57,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/529f16694e9b4d958836f1351215da39 is 50, key is test_row_0/B:col10/1731191996983/Put/seqid=0 2024-11-09T22:39:57,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741892_1068 (size=12251) 2024-11-09T22:39:57,430 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/529f16694e9b4d958836f1351215da39 2024-11-09T22:39:57,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192057433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192057439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/5707eb32a1bd46398be9e6d68e6b367e is 50, key is test_row_0/C:col10/1731191996983/Put/seqid=0 2024-11-09T22:39:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192057438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741893_1069 (size=12251) 2024-11-09T22:39:57,466 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/5707eb32a1bd46398be9e6d68e6b367e 2024-11-09T22:39:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/83856ef206f4421b9a5ee6824c7663af as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/83856ef206f4421b9a5ee6824c7663af 2024-11-09T22:39:57,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/83856ef206f4421b9a5ee6824c7663af, entries=150, sequenceid=262, filesize=12.0 K 2024-11-09T22:39:57,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/529f16694e9b4d958836f1351215da39 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/529f16694e9b4d958836f1351215da39 2024-11-09T22:39:57,494 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/529f16694e9b4d958836f1351215da39, entries=150, sequenceid=262, filesize=12.0 K 2024-11-09T22:39:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/5707eb32a1bd46398be9e6d68e6b367e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5707eb32a1bd46398be9e6d68e6b367e 2024-11-09T22:39:57,507 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5707eb32a1bd46398be9e6d68e6b367e, entries=150, sequenceid=262, filesize=12.0 K 2024-11-09T22:39:57,509 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1f18320792a575fd182c9198939e0c16 in 221ms, sequenceid=262, compaction requested=true 2024-11-09T22:39:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-09T22:39:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-09T22:39:57,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-09T22:39:57,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 537 msec 2024-11-09T22:39:57,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 558 msec 2024-11-09T22:39:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:57,551 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:39:57,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:39:57,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/8acfa6c3fecd4aafba919e324cceb2d9 is 50, key is test_row_0/A:col10/1731191997436/Put/seqid=0 2024-11-09T22:39:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192057575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192057576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192057581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T22:39:57,595 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:39:57,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741894_1070 (size=14741) 2024-11-09T22:39:57,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:39:57,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/8acfa6c3fecd4aafba919e324cceb2d9 2024-11-09T22:39:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-09T22:39:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T22:39:57,600 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:39:57,607 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:39:57,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:39:57,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/c1b77bae244d471f93127d4698bc8fd9 is 50, key is test_row_0/B:col10/1731191997436/Put/seqid=0 2024-11-09T22:39:57,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192057685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192057685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741895_1071 (size=12301) 2024-11-09T22:39:57,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192057697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T22:39:57,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T22:39:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:57,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192057897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192057897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:57,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T22:39:57,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:57,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T22:39:57,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192057924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T22:39:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/c1b77bae244d471f93127d4698bc8fd9 2024-11-09T22:39:58,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/5c3b554bbc1a40d58226aac05b75f031 is 50, key is test_row_0/C:col10/1731191997436/Put/seqid=0 2024-11-09T22:39:58,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741896_1072 (size=12301) 2024-11-09T22:39:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192058205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192058205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T22:39:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T22:39:58,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192058236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T22:39:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731192058415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,423 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 1f18320792a575fd182c9198939e0c16 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:39:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731192058425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,432 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1f18320792a575fd182c9198939e0c16 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8312 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:39:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T22:39:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:39:58,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/5c3b554bbc1a40d58226aac05b75f031 2024-11-09T22:39:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/8acfa6c3fecd4aafba919e324cceb2d9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8acfa6c3fecd4aafba919e324cceb2d9 2024-11-09T22:39:58,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8acfa6c3fecd4aafba919e324cceb2d9, entries=200, sequenceid=288, filesize=14.4 K 2024-11-09T22:39:58,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/c1b77bae244d471f93127d4698bc8fd9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/c1b77bae244d471f93127d4698bc8fd9 2024-11-09T22:39:58,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/c1b77bae244d471f93127d4698bc8fd9, entries=150, sequenceid=288, filesize=12.0 K 2024-11-09T22:39:58,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/5c3b554bbc1a40d58226aac05b75f031 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5c3b554bbc1a40d58226aac05b75f031 2024-11-09T22:39:58,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5c3b554bbc1a40d58226aac05b75f031, entries=150, sequenceid=288, filesize=12.0 K 2024-11-09T22:39:58,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1f18320792a575fd182c9198939e0c16 in 1067ms, sequenceid=288, compaction requested=true 2024-11-09T22:39:58,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:58,620 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:39:58,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:39:58,622 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:39:58,622 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:39:58,622 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,622 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8c948f48318f496289a0a1c3a2bc056a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/93d23f31e1b6408581054dd26f4ac977, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/83856ef206f4421b9a5ee6824c7663af, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8acfa6c3fecd4aafba919e324cceb2d9] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=55.3 K 2024-11-09T22:39:58,623 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c948f48318f496289a0a1c3a2bc056a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731191994013 2024-11-09T22:39:58,623 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d23f31e1b6408581054dd26f4ac977, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731191996244 2024-11-09T22:39:58,624 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83856ef206f4421b9a5ee6824c7663af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731191996969 2024-11-09T22:39:58,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:58,624 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8acfa6c3fecd4aafba919e324cceb2d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731191997424 2024-11-09T22:39:58,626 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:39:58,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:39:58,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:58,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:39:58,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:58,628 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:39:58,628 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:39:58,628 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,628 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9801ca83991b49a79c1c5c86808a9068, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8e83423c8643458f98831b0c36635012, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/529f16694e9b4d958836f1351215da39, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/c1b77bae244d471f93127d4698bc8fd9] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=48.2 K 2024-11-09T22:39:58,629 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9801ca83991b49a79c1c5c86808a9068, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731191994013 2024-11-09T22:39:58,629 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e83423c8643458f98831b0c36635012, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731191996274 2024-11-09T22:39:58,630 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 529f16694e9b4d958836f1351215da39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731191996969 2024-11-09T22:39:58,631 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b77bae244d471f93127d4698bc8fd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731191997424 2024-11-09T22:39:58,653 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:58,653 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/e08ac95500464889867465ec483160bc is 50, key is test_row_0/A:col10/1731191997436/Put/seqid=0 2024-11-09T22:39:58,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741897_1073 (size=12983) 2024-11-09T22:39:58,690 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:58,691 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/bc0e81359939422fa20b9ab2bd95907f is 50, key is test_row_0/B:col10/1731191997436/Put/seqid=0 2024-11-09T22:39:58,698 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/e08ac95500464889867465ec483160bc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e08ac95500464889867465ec483160bc 2024-11-09T22:39:58,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T22:39:58,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,702 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:39:58,715 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into e08ac95500464889867465ec483160bc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:58,715 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:58,715 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=12, startTime=1731191998619; duration=0sec 2024-11-09T22:39:58,715 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:39:58,716 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:39:58,716 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:39:58,718 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:39:58,718 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:39:58,719 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:58,719 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/99fb485527014dc099c86b609b70ebdc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/e28a1fb7dbc542529e59e711ddf7c62d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5707eb32a1bd46398be9e6d68e6b367e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5c3b554bbc1a40d58226aac05b75f031] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=48.2 K 2024-11-09T22:39:58,721 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99fb485527014dc099c86b609b70ebdc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731191994013 2024-11-09T22:39:58,722 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e28a1fb7dbc542529e59e711ddf7c62d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731191996274 2024-11-09T22:39:58,722 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5707eb32a1bd46398be9e6d68e6b367e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731191996969 2024-11-09T22:39:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:58,723 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c3b554bbc1a40d58226aac05b75f031, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731191997424 2024-11-09T22:39:58,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:58,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/37b3b8e6c68d44f98fcd009566236264 is 50, key is test_row_0/A:col10/1731191997578/Put/seqid=0 2024-11-09T22:39:58,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741898_1074 (size=12983) 2024-11-09T22:39:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T22:39:58,750 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/bc0e81359939422fa20b9ab2bd95907f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bc0e81359939422fa20b9ab2bd95907f 2024-11-09T22:39:58,761 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into bc0e81359939422fa20b9ab2bd95907f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:58,761 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:58,761 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=12, startTime=1731191998626; duration=0sec 2024-11-09T22:39:58,761 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:58,761 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:39:58,766 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#63 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:39:58,766 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/4cec082d8fe0450c9c608322e2c04b3a is 50, key is test_row_0/C:col10/1731191997436/Put/seqid=0 2024-11-09T22:39:58,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741899_1075 (size=12301) 2024-11-09T22:39:58,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741900_1076 (size=12983) 2024-11-09T22:39:58,815 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/4cec082d8fe0450c9c608322e2c04b3a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4cec082d8fe0450c9c608322e2c04b3a 2024-11-09T22:39:58,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192058815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192058823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192058824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,834 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into 4cec082d8fe0450c9c608322e2c04b3a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:39:58,834 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:58,835 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=12, startTime=1731191998627; duration=0sec 2024-11-09T22:39:58,835 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:39:58,835 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:39:58,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192058925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192058934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192058935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192059135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192059144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192059145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,199 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/37b3b8e6c68d44f98fcd009566236264 2024-11-09T22:39:59,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/651a3d08b879457abee9e7cf3645db71 is 50, key is test_row_0/B:col10/1731191997578/Put/seqid=0 2024-11-09T22:39:59,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741901_1077 (size=12301) 2024-11-09T22:39:59,239 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/651a3d08b879457abee9e7cf3645db71 2024-11-09T22:39:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/6abb3e6650e740119dd5fa90c9469a82 is 50, key is test_row_0/C:col10/1731191997578/Put/seqid=0 2024-11-09T22:39:59,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741902_1078 (size=12301) 2024-11-09T22:39:59,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192059445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192059455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192059455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,672 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/6abb3e6650e740119dd5fa90c9469a82 2024-11-09T22:39:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/37b3b8e6c68d44f98fcd009566236264 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/37b3b8e6c68d44f98fcd009566236264 2024-11-09T22:39:59,703 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/37b3b8e6c68d44f98fcd009566236264, entries=150, sequenceid=299, filesize=12.0 K 2024-11-09T22:39:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/651a3d08b879457abee9e7cf3645db71 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/651a3d08b879457abee9e7cf3645db71 2024-11-09T22:39:59,712 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/651a3d08b879457abee9e7cf3645db71, entries=150, sequenceid=299, filesize=12.0 K 2024-11-09T22:39:59,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/6abb3e6650e740119dd5fa90c9469a82 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/6abb3e6650e740119dd5fa90c9469a82 2024-11-09T22:39:59,731 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/6abb3e6650e740119dd5fa90c9469a82, entries=150, sequenceid=299, filesize=12.0 K 2024-11-09T22:39:59,732 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1f18320792a575fd182c9198939e0c16 in 1030ms, sequenceid=299, compaction requested=false 2024-11-09T22:39:59,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:39:59,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:59,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-09T22:39:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-09T22:39:59,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-09T22:39:59,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1280 sec 2024-11-09T22:39:59,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.1480 sec 2024-11-09T22:39:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T22:39:59,756 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:39:59,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:39:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-09T22:39:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T22:39:59,763 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:39:59,765 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:39:59,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:39:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T22:39:59,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T22:39:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:39:59,920 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T22:39:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/520f269af7614adcb28fe54294cde0b3 is 50, key is test_row_0/A:col10/1731191998815/Put/seqid=0 2024-11-09T22:39:59,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:39:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:39:59,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741903_1079 (size=12301) 2024-11-09T22:39:59,970 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/520f269af7614adcb28fe54294cde0b3 2024-11-09T22:39:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192059973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192059977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:39:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192059979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:39:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:39:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:39:59,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/d2cd6ee0e9c147e6918d37c1c198c36d is 50, key is test_row_0/B:col10/1731191998815/Put/seqid=0 2024-11-09T22:40:00,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741904_1080 (size=12301) 2024-11-09T22:40:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T22:40:00,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192060086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192060095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192060095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192060295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192060305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192060305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T22:40:00,406 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/d2cd6ee0e9c147e6918d37c1c198c36d 2024-11-09T22:40:00,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/2bdb018edbba4631ae6259bfb094497a is 50, key is test_row_0/C:col10/1731191998815/Put/seqid=0 2024-11-09T22:40:00,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741905_1081 (size=12301) 2024-11-09T22:40:00,445 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/2bdb018edbba4631ae6259bfb094497a 2024-11-09T22:40:00,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/520f269af7614adcb28fe54294cde0b3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/520f269af7614adcb28fe54294cde0b3 2024-11-09T22:40:00,463 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/520f269af7614adcb28fe54294cde0b3, entries=150, sequenceid=328, filesize=12.0 K 2024-11-09T22:40:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/d2cd6ee0e9c147e6918d37c1c198c36d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2cd6ee0e9c147e6918d37c1c198c36d 2024-11-09T22:40:00,472 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2cd6ee0e9c147e6918d37c1c198c36d, entries=150, sequenceid=328, filesize=12.0 K 2024-11-09T22:40:00,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/2bdb018edbba4631ae6259bfb094497a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/2bdb018edbba4631ae6259bfb094497a 2024-11-09T22:40:00,494 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/2bdb018edbba4631ae6259bfb094497a, entries=150, sequenceid=328, filesize=12.0 K 2024-11-09T22:40:00,507 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1f18320792a575fd182c9198939e0c16 in 586ms, sequenceid=328, compaction requested=true 2024-11-09T22:40:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-09T22:40:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-09T22:40:00,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T22:40:00,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 744 msec 2024-11-09T22:40:00,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 758 msec 2024-11-09T22:40:00,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:00,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:40:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:00,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/cae4490519b049248689c01084e8dadf is 50, key is test_row_0/A:col10/1731192000606/Put/seqid=0 2024-11-09T22:40:00,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741906_1082 (size=14741) 2024-11-09T22:40:00,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/cae4490519b049248689c01084e8dadf 2024-11-09T22:40:00,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4a81a5e8542a4cf394863c68d06a3ad1 is 50, key is test_row_0/B:col10/1731192000606/Put/seqid=0 2024-11-09T22:40:00,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192060681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192060689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192060689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741907_1083 (size=12301) 2024-11-09T22:40:00,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192060795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192060805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192060805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T22:40:00,895 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:00,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-09T22:40:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T22:40:00,898 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:00,899 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:00,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T22:40:01,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192061005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192061015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192061015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T22:40:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4a81a5e8542a4cf394863c68d06a3ad1 2024-11-09T22:40:01,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/504037c5634c416e8b0a66082b5e2f5f is 50, key is test_row_0/C:col10/1731192000606/Put/seqid=0 2024-11-09T22:40:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741908_1084 (size=12301) 2024-11-09T22:40:01,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/504037c5634c416e8b0a66082b5e2f5f 2024-11-09T22:40:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/cae4490519b049248689c01084e8dadf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/cae4490519b049248689c01084e8dadf 2024-11-09T22:40:01,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/cae4490519b049248689c01084e8dadf, entries=200, sequenceid=339, filesize=14.4 K 2024-11-09T22:40:01,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4a81a5e8542a4cf394863c68d06a3ad1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4a81a5e8542a4cf394863c68d06a3ad1 2024-11-09T22:40:01,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4a81a5e8542a4cf394863c68d06a3ad1, entries=150, sequenceid=339, filesize=12.0 K 2024-11-09T22:40:01,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/504037c5634c416e8b0a66082b5e2f5f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/504037c5634c416e8b0a66082b5e2f5f 2024-11-09T22:40:01,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/504037c5634c416e8b0a66082b5e2f5f, entries=150, sequenceid=339, filesize=12.0 K 2024-11-09T22:40:01,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1f18320792a575fd182c9198939e0c16 in 549ms, sequenceid=339, compaction requested=true 2024-11-09T22:40:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:01,156 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:40:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:01,156 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:40:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:01,158 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:40:01,158 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:40:01,158 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:40:01,158 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:40:01,158 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,158 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,158 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bc0e81359939422fa20b9ab2bd95907f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/651a3d08b879457abee9e7cf3645db71, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2cd6ee0e9c147e6918d37c1c198c36d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4a81a5e8542a4cf394863c68d06a3ad1] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=48.7 K 2024-11-09T22:40:01,158 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e08ac95500464889867465ec483160bc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/37b3b8e6c68d44f98fcd009566236264, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/520f269af7614adcb28fe54294cde0b3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/cae4490519b049248689c01084e8dadf] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=51.1 K 2024-11-09T22:40:01,158 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting bc0e81359939422fa20b9ab2bd95907f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731191997424 2024-11-09T22:40:01,159 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e08ac95500464889867465ec483160bc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731191997424 2024-11-09T22:40:01,159 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b3b8e6c68d44f98fcd009566236264, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731191997560 2024-11-09T22:40:01,159 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 651a3d08b879457abee9e7cf3645db71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731191997560 2024-11-09T22:40:01,159 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 520f269af7614adcb28fe54294cde0b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731191998803 2024-11-09T22:40:01,159 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d2cd6ee0e9c147e6918d37c1c198c36d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731191998803 2024-11-09T22:40:01,160 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a81a5e8542a4cf394863c68d06a3ad1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731191999973 2024-11-09T22:40:01,160 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting cae4490519b049248689c01084e8dadf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731191999967 2024-11-09T22:40:01,172 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:01,172 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/52e609b88b454b4d9342682e1c05d9cb is 50, key is test_row_0/A:col10/1731192000606/Put/seqid=0 2024-11-09T22:40:01,174 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#73 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:01,175 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/bebb6e062ad140ce9c3659a67cf744ed is 50, key is test_row_0/B:col10/1731192000606/Put/seqid=0 2024-11-09T22:40:01,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741909_1085 (size=13119) 2024-11-09T22:40:01,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741910_1086 (size=13119) 2024-11-09T22:40:01,184 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/52e609b88b454b4d9342682e1c05d9cb as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/52e609b88b454b4d9342682e1c05d9cb 2024-11-09T22:40:01,191 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into 52e609b88b454b4d9342682e1c05d9cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:01,191 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:01,191 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=12, startTime=1731192001156; duration=0sec 2024-11-09T22:40:01,191 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:01,191 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:40:01,191 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:40:01,193 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:40:01,193 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:40:01,193 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,194 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4cec082d8fe0450c9c608322e2c04b3a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/6abb3e6650e740119dd5fa90c9469a82, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/2bdb018edbba4631ae6259bfb094497a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/504037c5634c416e8b0a66082b5e2f5f] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=48.7 K 2024-11-09T22:40:01,194 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cec082d8fe0450c9c608322e2c04b3a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731191997424 2024-11-09T22:40:01,194 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6abb3e6650e740119dd5fa90c9469a82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731191997560 2024-11-09T22:40:01,194 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bdb018edbba4631ae6259bfb094497a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731191998803 2024-11-09T22:40:01,195 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 504037c5634c416e8b0a66082b5e2f5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731191999973 2024-11-09T22:40:01,206 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:01,207 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/3ff34409363f4d8480aa907388f4dcd9 is 50, key is test_row_0/C:col10/1731192000606/Put/seqid=0 2024-11-09T22:40:01,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T22:40:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,211 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T22:40:01,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741911_1087 (size=13119) 2024-11-09T22:40:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T22:40:01,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/de298dfe50b641c6b2a5f5f3c49f9fe8 is 50, key is test_row_0/A:col10/1731192000642/Put/seqid=0 2024-11-09T22:40:01,224 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/3ff34409363f4d8480aa907388f4dcd9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/3ff34409363f4d8480aa907388f4dcd9 2024-11-09T22:40:01,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741912_1088 (size=12301) 2024-11-09T22:40:01,235 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/de298dfe50b641c6b2a5f5f3c49f9fe8 2024-11-09T22:40:01,235 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into 3ff34409363f4d8480aa907388f4dcd9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:01,236 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:01,236 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=12, startTime=1731192001156; duration=0sec 2024-11-09T22:40:01,236 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:01,236 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:40:01,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4b8cd0e5d138450a955bd84d4f96a62d is 50, key is test_row_0/B:col10/1731192000642/Put/seqid=0 2024-11-09T22:40:01,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741913_1089 (size=12301) 2024-11-09T22:40:01,260 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4b8cd0e5d138450a955bd84d4f96a62d 2024-11-09T22:40:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/be44e331c9fb4f0f840b99df0b0a19f5 is 50, key is test_row_0/C:col10/1731192000642/Put/seqid=0 2024-11-09T22:40:01,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741914_1090 (size=12301) 2024-11-09T22:40:01,297 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/be44e331c9fb4f0f840b99df0b0a19f5 2024-11-09T22:40:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/de298dfe50b641c6b2a5f5f3c49f9fe8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/de298dfe50b641c6b2a5f5f3c49f9fe8 2024-11-09T22:40:01,318 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/de298dfe50b641c6b2a5f5f3c49f9fe8, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T22:40:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/4b8cd0e5d138450a955bd84d4f96a62d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4b8cd0e5d138450a955bd84d4f96a62d 2024-11-09T22:40:01,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:01,326 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4b8cd0e5d138450a955bd84d4f96a62d, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T22:40:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/be44e331c9fb4f0f840b99df0b0a19f5 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/be44e331c9fb4f0f840b99df0b0a19f5 2024-11-09T22:40:01,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192061336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192061337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192061339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,342 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/be44e331c9fb4f0f840b99df0b0a19f5, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T22:40:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,344 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1f18320792a575fd182c9198939e0c16 in 133ms, sequenceid=366, compaction requested=false 2024-11-09T22:40:01,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:01,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-09T22:40:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-09T22:40:01,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-09T22:40:01,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 448 msec 2024-11-09T22:40:01,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 456 msec 2024-11-09T22:40:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:01,449 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:01,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T22:40:01,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/02406a4df921482a86e0cd1b3e90c5e6 is 50, key is test_row_0/A:col10/1731192001329/Put/seqid=0 2024-11-09T22:40:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741915_1091 (size=12301) 2024-11-09T22:40:01,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/02406a4df921482a86e0cd1b3e90c5e6 2024-11-09T22:40:01,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/3fa808ee68b343b2a85eb0d916ec0b56 is 50, key is test_row_0/B:col10/1731192001329/Put/seqid=0 2024-11-09T22:40:01,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741916_1092 (size=12301) 2024-11-09T22:40:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T22:40:01,526 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:01,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-09T22:40:01,528 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T22:40:01,529 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:01,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:01,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192061545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192061546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192061549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,588 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/bebb6e062ad140ce9c3659a67cf744ed as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bebb6e062ad140ce9c3659a67cf744ed 2024-11-09T22:40:01,595 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into bebb6e062ad140ce9c3659a67cf744ed(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:01,595 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:01,595 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=12, startTime=1731192001156; duration=0sec 2024-11-09T22:40:01,595 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:01,595 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:40:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T22:40:01,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192061655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192061655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192061665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T22:40:01,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:01,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T22:40:01,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:01,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T22:40:01,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192061865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192061865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192061875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:01,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/3fa808ee68b343b2a85eb0d916ec0b56 2024-11-09T22:40:01,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/86837fbc46c143b9881a05f3ad539b4a is 50, key is test_row_0/C:col10/1731192001329/Put/seqid=0 2024-11-09T22:40:01,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741917_1093 (size=12301) 2024-11-09T22:40:01,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T22:40:01,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:01,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:01,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:02,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T22:40:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:02,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T22:40:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192062175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192062175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192062184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T22:40:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:02,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:02,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/86837fbc46c143b9881a05f3ad539b4a 2024-11-09T22:40:02,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/02406a4df921482a86e0cd1b3e90c5e6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/02406a4df921482a86e0cd1b3e90c5e6 2024-11-09T22:40:02,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/02406a4df921482a86e0cd1b3e90c5e6, entries=150, sequenceid=380, filesize=12.0 K 2024-11-09T22:40:02,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/3fa808ee68b343b2a85eb0d916ec0b56 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3fa808ee68b343b2a85eb0d916ec0b56 2024-11-09T22:40:02,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3fa808ee68b343b2a85eb0d916ec0b56, entries=150, sequenceid=380, filesize=12.0 K 2024-11-09T22:40:02,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/86837fbc46c143b9881a05f3ad539b4a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86837fbc46c143b9881a05f3ad539b4a 2024-11-09T22:40:02,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86837fbc46c143b9881a05f3ad539b4a, entries=150, sequenceid=380, filesize=12.0 K 2024-11-09T22:40:02,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1f18320792a575fd182c9198939e0c16 in 911ms, sequenceid=380, compaction requested=true 2024-11-09T22:40:02,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:02,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:02,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:02,360 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:02,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:02,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:02,360 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:02,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:02,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:02,361 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:02,362 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:02,362 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:40:02,362 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:40:02,362 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:02,362 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:02,362 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bebb6e062ad140ce9c3659a67cf744ed, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4b8cd0e5d138450a955bd84d4f96a62d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3fa808ee68b343b2a85eb0d916ec0b56] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=36.8 K 2024-11-09T22:40:02,362 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/52e609b88b454b4d9342682e1c05d9cb, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/de298dfe50b641c6b2a5f5f3c49f9fe8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/02406a4df921482a86e0cd1b3e90c5e6] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=36.8 K 2024-11-09T22:40:02,362 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52e609b88b454b4d9342682e1c05d9cb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731191999973 2024-11-09T22:40:02,362 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting bebb6e062ad140ce9c3659a67cf744ed, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731191999973 2024-11-09T22:40:02,363 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting de298dfe50b641c6b2a5f5f3c49f9fe8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731192000642 2024-11-09T22:40:02,363 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b8cd0e5d138450a955bd84d4f96a62d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731192000642 2024-11-09T22:40:02,363 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02406a4df921482a86e0cd1b3e90c5e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731192001329 2024-11-09T22:40:02,363 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa808ee68b343b2a85eb0d916ec0b56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731192001329 2024-11-09T22:40:02,377 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:02,378 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/d2a3e020421d4efd9d89d84a7ae0846e is 50, key is test_row_0/B:col10/1731192001329/Put/seqid=0 2024-11-09T22:40:02,393 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:02,394 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/bbc2d9aa07aa4535a23872dff9f9e658 is 50, key is test_row_0/A:col10/1731192001329/Put/seqid=0 2024-11-09T22:40:02,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741918_1094 (size=13221) 2024-11-09T22:40:02,412 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/d2a3e020421d4efd9d89d84a7ae0846e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2a3e020421d4efd9d89d84a7ae0846e 2024-11-09T22:40:02,426 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into d2a3e020421d4efd9d89d84a7ae0846e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:02,426 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:02,426 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=13, startTime=1731192002360; duration=0sec 2024-11-09T22:40:02,426 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:02,427 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:40:02,429 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:02,431 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:02,431 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:40:02,431 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:02,431 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/3ff34409363f4d8480aa907388f4dcd9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/be44e331c9fb4f0f840b99df0b0a19f5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86837fbc46c143b9881a05f3ad539b4a] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=36.8 K 2024-11-09T22:40:02,432 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ff34409363f4d8480aa907388f4dcd9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731191999973 2024-11-09T22:40:02,433 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting be44e331c9fb4f0f840b99df0b0a19f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731192000642 2024-11-09T22:40:02,434 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 86837fbc46c143b9881a05f3ad539b4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731192001329 2024-11-09T22:40:02,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741919_1095 (size=13221) 2024-11-09T22:40:02,447 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/bbc2d9aa07aa4535a23872dff9f9e658 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bbc2d9aa07aa4535a23872dff9f9e658 2024-11-09T22:40:02,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T22:40:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:02,466 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T22:40:02,468 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into bbc2d9aa07aa4535a23872dff9f9e658(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:02,468 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:02,468 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=13, startTime=1731192002360; duration=0sec 2024-11-09T22:40:02,468 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:02,468 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:40:02,471 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:02,472 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/21530fa6541046cdadb9881092392ba0 is 50, key is test_row_0/C:col10/1731192001329/Put/seqid=0 2024-11-09T22:40:02,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/69af8a046198435281b4cc58215c8200 is 50, key is test_row_0/A:col10/1731192001540/Put/seqid=0 2024-11-09T22:40:02,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741920_1096 (size=13221) 2024-11-09T22:40:02,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741921_1097 (size=12301) 2024-11-09T22:40:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T22:40:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:02,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192062698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192062699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192062701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192062805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192062805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192062805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:02,894 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/69af8a046198435281b4cc58215c8200 2024-11-09T22:40:02,898 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/21530fa6541046cdadb9881092392ba0 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/21530fa6541046cdadb9881092392ba0 2024-11-09T22:40:02,918 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into 21530fa6541046cdadb9881092392ba0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:02,919 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:02,919 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=13, startTime=1731192002360; duration=0sec 2024-11-09T22:40:02,919 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:02,919 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:40:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/10176ed925754bf09f4ffca6d31dfff2 is 50, key is test_row_0/B:col10/1731192001540/Put/seqid=0 2024-11-09T22:40:02,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741922_1098 (size=12301) 2024-11-09T22:40:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192063015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192063015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192063015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192063325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192063325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192063325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,377 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/10176ed925754bf09f4ffca6d31dfff2 2024-11-09T22:40:03,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/66cb2c6741304bf1a9a39e36b16b7ebf is 50, key is test_row_0/C:col10/1731192001540/Put/seqid=0 2024-11-09T22:40:03,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741923_1099 (size=12301) 2024-11-09T22:40:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T22:40:03,829 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/66cb2c6741304bf1a9a39e36b16b7ebf 2024-11-09T22:40:03,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/69af8a046198435281b4cc58215c8200 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/69af8a046198435281b4cc58215c8200 2024-11-09T22:40:03,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192063836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192063836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,840 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/69af8a046198435281b4cc58215c8200, entries=150, sequenceid=405, filesize=12.0 K 2024-11-09T22:40:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/10176ed925754bf09f4ffca6d31dfff2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/10176ed925754bf09f4ffca6d31dfff2 2024-11-09T22:40:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192063845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:03,849 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/10176ed925754bf09f4ffca6d31dfff2, entries=150, sequenceid=405, filesize=12.0 K 2024-11-09T22:40:03,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/66cb2c6741304bf1a9a39e36b16b7ebf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/66cb2c6741304bf1a9a39e36b16b7ebf 2024-11-09T22:40:03,856 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/66cb2c6741304bf1a9a39e36b16b7ebf, entries=150, sequenceid=405, filesize=12.0 K 2024-11-09T22:40:03,857 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1f18320792a575fd182c9198939e0c16 in 1391ms, sequenceid=405, compaction requested=false 2024-11-09T22:40:03,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:03,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:03,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-09T22:40:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-09T22:40:03,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-09T22:40:03,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3300 sec 2024-11-09T22:40:03,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.3360 sec 2024-11-09T22:40:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:04,846 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:04,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T22:40:04,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/156d81d22610441da8a81c6c41032de7 is 50, key is test_row_0/A:col10/1731192002696/Put/seqid=0 2024-11-09T22:40:04,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741924_1100 (size=14741) 2024-11-09T22:40:04,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/156d81d22610441da8a81c6c41032de7 2024-11-09T22:40:04,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/fb7e60d6507d4ee688c2c14b91b17ec3 is 50, key is test_row_0/B:col10/1731192002696/Put/seqid=0 2024-11-09T22:40:04,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741925_1101 (size=12301) 2024-11-09T22:40:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192064885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192064886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:04,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192064887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192064995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192064995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192064999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:05,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192065204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192065205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192065205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:05,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/fb7e60d6507d4ee688c2c14b91b17ec3 2024-11-09T22:40:05,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/8d7ccf1be5f744bc8c231941d666cb0f is 50, key is test_row_0/C:col10/1731192002696/Put/seqid=0 2024-11-09T22:40:05,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741926_1102 (size=12301) 2024-11-09T22:40:05,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192065515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:05,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192065515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:05,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192065515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T22:40:05,695 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:05,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-09T22:40:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T22:40:05,697 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:05,698 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:05,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:05,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/8d7ccf1be5f744bc8c231941d666cb0f 2024-11-09T22:40:05,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/156d81d22610441da8a81c6c41032de7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/156d81d22610441da8a81c6c41032de7 2024-11-09T22:40:05,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/156d81d22610441da8a81c6c41032de7, entries=200, sequenceid=421, filesize=14.4 K 2024-11-09T22:40:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/fb7e60d6507d4ee688c2c14b91b17ec3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/fb7e60d6507d4ee688c2c14b91b17ec3 2024-11-09T22:40:05,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/fb7e60d6507d4ee688c2c14b91b17ec3, entries=150, sequenceid=421, filesize=12.0 K 2024-11-09T22:40:05,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/8d7ccf1be5f744bc8c231941d666cb0f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/8d7ccf1be5f744bc8c231941d666cb0f 2024-11-09T22:40:05,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/8d7ccf1be5f744bc8c231941d666cb0f, entries=150, sequenceid=421, filesize=12.0 K 2024-11-09T22:40:05,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1f18320792a575fd182c9198939e0c16 in 890ms, sequenceid=421, compaction requested=true 2024-11-09T22:40:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:05,736 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:05,736 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:05,737 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:05,737 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:05,737 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:40:05,737 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:40:05,738 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:05,738 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:05,738 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bbc2d9aa07aa4535a23872dff9f9e658, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/69af8a046198435281b4cc58215c8200, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/156d81d22610441da8a81c6c41032de7] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=39.3 K 2024-11-09T22:40:05,738 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2a3e020421d4efd9d89d84a7ae0846e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/10176ed925754bf09f4ffca6d31dfff2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/fb7e60d6507d4ee688c2c14b91b17ec3] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=36.9 K 2024-11-09T22:40:05,738 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbc2d9aa07aa4535a23872dff9f9e658, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731192001329 2024-11-09T22:40:05,738 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a3e020421d4efd9d89d84a7ae0846e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731192001329 2024-11-09T22:40:05,738 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69af8a046198435281b4cc58215c8200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731192001540 2024-11-09T22:40:05,739 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 10176ed925754bf09f4ffca6d31dfff2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731192001540 2024-11-09T22:40:05,739 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 156d81d22610441da8a81c6c41032de7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731192002696 2024-11-09T22:40:05,739 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting fb7e60d6507d4ee688c2c14b91b17ec3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731192002696 2024-11-09T22:40:05,747 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:05,748 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/8702c97f5d3c4a01ae1cd67c90a943bc is 50, key is test_row_0/B:col10/1731192002696/Put/seqid=0 2024-11-09T22:40:05,750 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#91 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:05,751 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/24910344e4cf43a387315884c56f2a0e is 50, key is test_row_0/A:col10/1731192002696/Put/seqid=0 2024-11-09T22:40:05,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741928_1104 (size=13323) 2024-11-09T22:40:05,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741927_1103 (size=13323) 2024-11-09T22:40:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T22:40:05,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-09T22:40:05,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:05,851 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T22:40:05,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/fd711c181f044b5c9ba877029806d730 is 50, key is test_row_0/A:col10/1731192004885/Put/seqid=0 2024-11-09T22:40:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741929_1105 (size=12301) 2024-11-09T22:40:05,868 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/fd711c181f044b5c9ba877029806d730 2024-11-09T22:40:05,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e361e2ab494f4f18bc132b703d34edbe is 50, key is test_row_0/B:col10/1731192004885/Put/seqid=0 2024-11-09T22:40:05,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741930_1106 (size=12301) 2024-11-09T22:40:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T22:40:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:06,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:06,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192066038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192066039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192066047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192066154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192066155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192066155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,164 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/24910344e4cf43a387315884c56f2a0e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/24910344e4cf43a387315884c56f2a0e 2024-11-09T22:40:06,173 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/8702c97f5d3c4a01ae1cd67c90a943bc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8702c97f5d3c4a01ae1cd67c90a943bc 2024-11-09T22:40:06,175 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into 24910344e4cf43a387315884c56f2a0e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:06,175 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:06,175 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=13, startTime=1731192005736; duration=0sec 2024-11-09T22:40:06,176 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:06,176 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:40:06,176 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:06,177 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:06,177 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:40:06,177 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:06,177 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/21530fa6541046cdadb9881092392ba0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/66cb2c6741304bf1a9a39e36b16b7ebf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/8d7ccf1be5f744bc8c231941d666cb0f] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=36.9 K 2024-11-09T22:40:06,178 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21530fa6541046cdadb9881092392ba0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731192001329 2024-11-09T22:40:06,179 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66cb2c6741304bf1a9a39e36b16b7ebf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731192001540 2024-11-09T22:40:06,179 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d7ccf1be5f744bc8c231941d666cb0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731192002696 2024-11-09T22:40:06,181 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into 8702c97f5d3c4a01ae1cd67c90a943bc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:06,181 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:06,181 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=13, startTime=1731192005736; duration=0sec 2024-11-09T22:40:06,181 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:06,181 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:40:06,190 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:06,191 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/b8dd65a3360045a4b58665e426430408 is 50, key is test_row_0/C:col10/1731192002696/Put/seqid=0 2024-11-09T22:40:06,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741931_1107 (size=13323) 2024-11-09T22:40:06,209 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/b8dd65a3360045a4b58665e426430408 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/b8dd65a3360045a4b58665e426430408 2024-11-09T22:40:06,215 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into b8dd65a3360045a4b58665e426430408(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:06,215 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:06,215 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=13, startTime=1731192005736; duration=0sec 2024-11-09T22:40:06,216 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:06,216 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:40:06,285 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e361e2ab494f4f18bc132b703d34edbe 2024-11-09T22:40:06,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/772b11179a594469b728e999fc1c8cb2 is 50, key is test_row_0/C:col10/1731192004885/Put/seqid=0 2024-11-09T22:40:06,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741932_1108 (size=12301) 2024-11-09T22:40:06,302 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/772b11179a594469b728e999fc1c8cb2 2024-11-09T22:40:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/fd711c181f044b5c9ba877029806d730 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/fd711c181f044b5c9ba877029806d730 2024-11-09T22:40:06,316 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/fd711c181f044b5c9ba877029806d730, entries=150, sequenceid=443, filesize=12.0 K 2024-11-09T22:40:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e361e2ab494f4f18bc132b703d34edbe as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e361e2ab494f4f18bc132b703d34edbe 2024-11-09T22:40:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T22:40:06,328 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e361e2ab494f4f18bc132b703d34edbe, entries=150, sequenceid=443, filesize=12.0 K 2024-11-09T22:40:06,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/772b11179a594469b728e999fc1c8cb2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/772b11179a594469b728e999fc1c8cb2 2024-11-09T22:40:06,335 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/772b11179a594469b728e999fc1c8cb2, entries=150, sequenceid=443, filesize=12.0 K 2024-11-09T22:40:06,336 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1f18320792a575fd182c9198939e0c16 in 485ms, sequenceid=443, compaction requested=false 2024-11-09T22:40:06,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:06,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:06,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-09T22:40:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-09T22:40:06,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T22:40:06,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 639 msec 2024-11-09T22:40:06,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 645 msec 2024-11-09T22:40:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:06,367 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:06,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T22:40:06,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4edd99b9862b4dc095468ee5725df007 is 50, key is test_row_0/A:col10/1731192006037/Put/seqid=0 2024-11-09T22:40:06,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741933_1109 (size=17181) 2024-11-09T22:40:06,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192066393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192066393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192066394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192066505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192066505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192066505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192066715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192066715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192066715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:06,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4edd99b9862b4dc095468ee5725df007 2024-11-09T22:40:06,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e6b09c84e052464fab0de1d6eef0738a is 50, key is test_row_0/B:col10/1731192006037/Put/seqid=0 2024-11-09T22:40:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T22:40:06,835 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:06,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-09T22:40:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T22:40:06,838 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:06,839 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:06,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:06,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741934_1110 (size=12301) 2024-11-09T22:40:06,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e6b09c84e052464fab0de1d6eef0738a 2024-11-09T22:40:06,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/55d65950332343d0a2801d5c029bccb9 is 50, key is test_row_0/C:col10/1731192006037/Put/seqid=0 2024-11-09T22:40:06,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741935_1111 (size=12301) 2024-11-09T22:40:06,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/55d65950332343d0a2801d5c029bccb9 2024-11-09T22:40:06,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/4edd99b9862b4dc095468ee5725df007 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4edd99b9862b4dc095468ee5725df007 2024-11-09T22:40:06,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4edd99b9862b4dc095468ee5725df007, entries=250, sequenceid=462, filesize=16.8 K 2024-11-09T22:40:06,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/e6b09c84e052464fab0de1d6eef0738a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e6b09c84e052464fab0de1d6eef0738a 2024-11-09T22:40:06,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e6b09c84e052464fab0de1d6eef0738a, entries=150, sequenceid=462, filesize=12.0 K 2024-11-09T22:40:06,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/55d65950332343d0a2801d5c029bccb9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/55d65950332343d0a2801d5c029bccb9 2024-11-09T22:40:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T22:40:06,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/55d65950332343d0a2801d5c029bccb9, entries=150, sequenceid=462, filesize=12.0 K 2024-11-09T22:40:06,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1f18320792a575fd182c9198939e0c16 in 582ms, sequenceid=462, compaction requested=true 2024-11-09T22:40:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:06,950 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:06,952 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:06,952 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/A is initiating minor compaction (all files) 2024-11-09T22:40:06,952 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/A in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:06,953 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/24910344e4cf43a387315884c56f2a0e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/fd711c181f044b5c9ba877029806d730, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4edd99b9862b4dc095468ee5725df007] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=41.8 K 2024-11-09T22:40:06,953 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24910344e4cf43a387315884c56f2a0e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731192002696 2024-11-09T22:40:06,954 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd711c181f044b5c9ba877029806d730, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731192004875 2024-11-09T22:40:06,955 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4edd99b9862b4dc095468ee5725df007, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731192006037 2024-11-09T22:40:06,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:06,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:06,957 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:06,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:06,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:06,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:06,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:06,962 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:06,962 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/B is initiating minor compaction (all files) 2024-11-09T22:40:06,962 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/B in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:06,962 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8702c97f5d3c4a01ae1cd67c90a943bc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e361e2ab494f4f18bc132b703d34edbe, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e6b09c84e052464fab0de1d6eef0738a] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=37.0 K 2024-11-09T22:40:06,963 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8702c97f5d3c4a01ae1cd67c90a943bc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731192002696 2024-11-09T22:40:06,966 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting e361e2ab494f4f18bc132b703d34edbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731192004875 2024-11-09T22:40:06,967 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b09c84e052464fab0de1d6eef0738a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731192006037 2024-11-09T22:40:06,981 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:06,982 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/7f88000d0755416f9f636e5986d5c83a is 50, key is test_row_0/A:col10/1731192006037/Put/seqid=0 2024-11-09T22:40:06,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-09T22:40:06,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:06,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T22:40:06,993 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:06,994 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/256c1f7e6bf74d2cbfda28f4c4550508 is 50, key is test_row_0/B:col10/1731192006037/Put/seqid=0 2024-11-09T22:40:07,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/927e3ad7210941f5a4eb7a205f18aa9f is 50, key is test_row_0/A:col10/1731192006392/Put/seqid=0 2024-11-09T22:40:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:07,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. as already flushing 2024-11-09T22:40:07,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741936_1112 (size=13425) 2024-11-09T22:40:07,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741937_1113 (size=13425) 2024-11-09T22:40:07,062 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/256c1f7e6bf74d2cbfda28f4c4550508 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/256c1f7e6bf74d2cbfda28f4c4550508 2024-11-09T22:40:07,071 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/B of 1f18320792a575fd182c9198939e0c16 into 256c1f7e6bf74d2cbfda28f4c4550508(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:07,071 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:07,071 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/B, priority=13, startTime=1731192006957; duration=0sec 2024-11-09T22:40:07,071 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:07,072 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:40:07,072 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:07,073 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:07,073 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1f18320792a575fd182c9198939e0c16/C is initiating minor compaction (all files) 2024-11-09T22:40:07,073 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f18320792a575fd182c9198939e0c16/C in TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:07,073 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/b8dd65a3360045a4b58665e426430408, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/772b11179a594469b728e999fc1c8cb2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/55d65950332343d0a2801d5c029bccb9] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp, totalSize=37.0 K 2024-11-09T22:40:07,074 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting b8dd65a3360045a4b58665e426430408, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731192002696 2024-11-09T22:40:07,074 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 772b11179a594469b728e999fc1c8cb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731192004875 2024-11-09T22:40:07,075 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 55d65950332343d0a2801d5c029bccb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731192006037 2024-11-09T22:40:07,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192067064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192067064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192067061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741938_1114 (size=12301) 2024-11-09T22:40:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,088 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f18320792a575fd182c9198939e0c16#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:07,089 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/86303de5f5454da5ad8a98838b4e834e is 50, key is test_row_0/C:col10/1731192006037/Put/seqid=0 2024-11-09T22:40:07,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741939_1115 (size=13425) 2024-11-09T22:40:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T22:40:07,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192067185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192067185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192067185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,371 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-09T22:40:07,372 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:07,377 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-09T22:40:07,377 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:07,378 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-09T22:40:07,378 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:07,379 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:07,379 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:07,380 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:07,380 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:07,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:07,381 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:07,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:07,381 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:07,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:07,384 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-09T22:40:07,384 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:07,384 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:07,384 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:07,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192067395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192067395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192067395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,438 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/7f88000d0755416f9f636e5986d5c83a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7f88000d0755416f9f636e5986d5c83a 2024-11-09T22:40:07,446 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/A of 1f18320792a575fd182c9198939e0c16 into 7f88000d0755416f9f636e5986d5c83a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:07,446 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:07,446 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/A, priority=13, startTime=1731192006949; duration=0sec 2024-11-09T22:40:07,446 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:07,446 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:40:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T22:40:07,479 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/927e3ad7210941f5a4eb7a205f18aa9f 2024-11-09T22:40:07,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/d5849d9b38b84d8cbcac1dc7278b2d01 is 50, key is test_row_0/B:col10/1731192006392/Put/seqid=0 2024-11-09T22:40:07,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741940_1116 (size=12301) 2024-11-09T22:40:07,516 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/86303de5f5454da5ad8a98838b4e834e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86303de5f5454da5ad8a98838b4e834e 2024-11-09T22:40:07,521 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f18320792a575fd182c9198939e0c16/C of 1f18320792a575fd182c9198939e0c16 into 86303de5f5454da5ad8a98838b4e834e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:07,521 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:07,521 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., storeName=1f18320792a575fd182c9198939e0c16/C, priority=13, startTime=1731192006957; duration=0sec 2024-11-09T22:40:07,521 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:07,521 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:40:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192067705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192067705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192067705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:07,893 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/d5849d9b38b84d8cbcac1dc7278b2d01 2024-11-09T22:40:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/c478918045e44d45987743bcae2cafee is 50, key is test_row_0/C:col10/1731192006392/Put/seqid=0 2024-11-09T22:40:07,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741941_1117 (size=12301) 2024-11-09T22:40:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T22:40:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49504 deadline: 1731192068215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49512 deadline: 1731192068215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49468 deadline: 1731192068215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:08,308 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/c478918045e44d45987743bcae2cafee 2024-11-09T22:40:08,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/927e3ad7210941f5a4eb7a205f18aa9f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/927e3ad7210941f5a4eb7a205f18aa9f 2024-11-09T22:40:08,320 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/927e3ad7210941f5a4eb7a205f18aa9f, entries=150, sequenceid=482, filesize=12.0 K 2024-11-09T22:40:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/d5849d9b38b84d8cbcac1dc7278b2d01 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d5849d9b38b84d8cbcac1dc7278b2d01 2024-11-09T22:40:08,336 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d5849d9b38b84d8cbcac1dc7278b2d01, entries=150, sequenceid=482, filesize=12.0 K 2024-11-09T22:40:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/c478918045e44d45987743bcae2cafee as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c478918045e44d45987743bcae2cafee 2024-11-09T22:40:08,345 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c478918045e44d45987743bcae2cafee, entries=150, sequenceid=482, filesize=12.0 K 2024-11-09T22:40:08,346 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1f18320792a575fd182c9198939e0c16 in 1354ms, sequenceid=482, compaction requested=false 2024-11-09T22:40:08,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:08,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:08,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-09T22:40:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-09T22:40:08,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-09T22:40:08,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-11-09T22:40:08,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.5160 sec 2024-11-09T22:40:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:08,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:08,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T22:40:08,516 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-09T22:40:08,516 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:08,516 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:08,517 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:08,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:08,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/afacd2aa984840e8ace4b94a84938732 is 50, key is test_row_0/A:col10/1731192008515/Put/seqid=0 2024-11-09T22:40:08,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741942_1118 (size=12301) 2024-11-09T22:40:08,527 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-09T22:40:08,527 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:08,527 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:08,527 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:08,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:08,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/afacd2aa984840e8ace4b94a84938732 2024-11-09T22:40:08,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/3b6b9e00f59d4c84aeffeb991574a285 is 50, key is test_row_0/B:col10/1731192008515/Put/seqid=0 2024-11-09T22:40:08,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741943_1119 (size=12301) 2024-11-09T22:40:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T22:40:08,985 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:09,228 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-09T22:40:09,228 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-09T22:40:09,229 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:09,229 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:09,229 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:09,229 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:09,229 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-09T22:40:09,229 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:09,229 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:09,229 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:09,229 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:09,229 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:09,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:09,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T22:40:09,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-09T22:40:09,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-09T22:40:09,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-09T22:40:09,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-09T22:40:09,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-09T22:40:09,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T22:40:09,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:09,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:09,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3050 2024-11-09T22:40:09,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3037 2024-11-09T22:40:09,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T22:40:09,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1392 2024-11-09T22:40:09,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4175 rows 2024-11-09T22:40:09,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1407 2024-11-09T22:40:09,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4218 rows 2024-11-09T22:40:09,232 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:40:09,232 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:40:09,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:09,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:09,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:09,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T22:40:09,244 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T22:40:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:09,267 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192009267"}]},"ts":"1731192009267"} 2024-11-09T22:40:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T22:40:09,269 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T22:40:09,269 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T22:40:09,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:40:09,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, UNASSIGN}] 2024-11-09T22:40:09,276 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, UNASSIGN 2024-11-09T22:40:09,277 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1f18320792a575fd182c9198939e0c16, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:09,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, UNASSIGN because future has completed 2024-11-09T22:40:09,280 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:40:09,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:40:09,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/3b6b9e00f59d4c84aeffeb991574a285 2024-11-09T22:40:09,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/83dcda76afbe4bd5bea87a14fd58f6a8 is 50, key is test_row_0/C:col10/1731192008515/Put/seqid=0 2024-11-09T22:40:09,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741944_1120 (size=12301) 2024-11-09T22:40:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T22:40:09,438 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:09,439 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:40:09,440 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 1f18320792a575fd182c9198939e0c16, disabling compactions & flushes 2024-11-09T22:40:09,440 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T22:40:09,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/83dcda76afbe4bd5bea87a14fd58f6a8 2024-11-09T22:40:09,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/afacd2aa984840e8ace4b94a84938732 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/afacd2aa984840e8ace4b94a84938732 2024-11-09T22:40:09,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/afacd2aa984840e8ace4b94a84938732, entries=150, sequenceid=502, filesize=12.0 K 2024-11-09T22:40:09,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/3b6b9e00f59d4c84aeffeb991574a285 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3b6b9e00f59d4c84aeffeb991574a285 2024-11-09T22:40:09,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3b6b9e00f59d4c84aeffeb991574a285, entries=150, sequenceid=502, filesize=12.0 K 2024-11-09T22:40:09,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/83dcda76afbe4bd5bea87a14fd58f6a8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/83dcda76afbe4bd5bea87a14fd58f6a8 2024-11-09T22:40:09,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/83dcda76afbe4bd5bea87a14fd58f6a8, entries=150, sequenceid=502, filesize=12.0 K 2024-11-09T22:40:09,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=26.84 KB/27480 for 1f18320792a575fd182c9198939e0c16 in 1270ms, sequenceid=502, compaction requested=true 2024-11-09T22:40:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f18320792a575fd182c9198939e0c16: 2024-11-09T22:40:09,786 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:09,786 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:09,786 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. after waiting 0 ms 2024-11-09T22:40:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:09,786 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f18320792a575fd182c9198939e0c16:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:09,786 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. because compaction request was cancelled 2024-11-09T22:40:09,786 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. because compaction request was cancelled 2024-11-09T22:40:09,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:09,786 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:B 2024-11-09T22:40:09,786 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:A 2024-11-09T22:40:09,786 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. because compaction request was cancelled 2024-11-09T22:40:09,786 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 1f18320792a575fd182c9198939e0c16 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T22:40:09,786 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f18320792a575fd182c9198939e0c16:C 2024-11-09T22:40:09,791 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/7802a57df3184bd8830b44cc47012a70 is 50, key is test_row_0/A:col10/1731192009226/Put/seqid=0 2024-11-09T22:40:09,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741945_1121 (size=9857) 2024-11-09T22:40:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T22:40:10,196 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/7802a57df3184bd8830b44cc47012a70 2024-11-09T22:40:10,210 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/7f1191fd60a2485995e0ae02f0cb40bc is 50, key is test_row_0/B:col10/1731192009226/Put/seqid=0 2024-11-09T22:40:10,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741946_1122 (size=9857) 2024-11-09T22:40:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T22:40:10,615 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/7f1191fd60a2485995e0ae02f0cb40bc 2024-11-09T22:40:10,624 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/eef75837d205487ab54fea046a10dfee is 50, key is test_row_0/C:col10/1731192009226/Put/seqid=0 2024-11-09T22:40:10,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741947_1123 (size=9857) 2024-11-09T22:40:11,028 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/eef75837d205487ab54fea046a10dfee 2024-11-09T22:40:11,033 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/A/7802a57df3184bd8830b44cc47012a70 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7802a57df3184bd8830b44cc47012a70 2024-11-09T22:40:11,039 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7802a57df3184bd8830b44cc47012a70, entries=100, sequenceid=509, filesize=9.6 K 2024-11-09T22:40:11,040 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/B/7f1191fd60a2485995e0ae02f0cb40bc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/7f1191fd60a2485995e0ae02f0cb40bc 2024-11-09T22:40:11,045 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/7f1191fd60a2485995e0ae02f0cb40bc, entries=100, sequenceid=509, filesize=9.6 K 2024-11-09T22:40:11,046 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/.tmp/C/eef75837d205487ab54fea046a10dfee as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/eef75837d205487ab54fea046a10dfee 2024-11-09T22:40:11,052 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/eef75837d205487ab54fea046a10dfee, entries=100, sequenceid=509, filesize=9.6 K 2024-11-09T22:40:11,053 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1f18320792a575fd182c9198939e0c16 in 1266ms, sequenceid=509, compaction requested=true 2024-11-09T22:40:11,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e9e167697c7b424e930e1d48813999fa, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bcc626afa89d4b6f8ab27178596b4a06, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/f7735160aae0429d966813e149a49a23, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/dea556156bca4db6a597f80c6d023cdd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1bb4ebcc5d4d4130bf7eeca69fae1222, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7bb1c56391064abe9a7c06489263a6d3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/2b77b618f1a24065bbdf6abc8f7a206f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1ac0311c59f2484ea2b88fe1a6611088, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4049b983abba4541849d882fe344ee92, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/9066252f5a56456a82a2079ac6dde416, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/c5d1203ade834f7c8a6293167a3ac983, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/a1bb68767ba346699dbbbb3d08dd5dfe, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4ef6297fe66d4b8b86f32586388627de, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/073194f889dd46eabdc979b4eb3597ff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6103b0172dc4470f973f529c76fda9f3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8c948f48318f496289a0a1c3a2bc056a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6179c18658a34d759d9101e00293d97d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/93d23f31e1b6408581054dd26f4ac977, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/83856ef206f4421b9a5ee6824c7663af, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8acfa6c3fecd4aafba919e324cceb2d9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e08ac95500464889867465ec483160bc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/37b3b8e6c68d44f98fcd009566236264, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/520f269af7614adcb28fe54294cde0b3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/cae4490519b049248689c01084e8dadf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/52e609b88b454b4d9342682e1c05d9cb, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/de298dfe50b641c6b2a5f5f3c49f9fe8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bbc2d9aa07aa4535a23872dff9f9e658, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/02406a4df921482a86e0cd1b3e90c5e6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/69af8a046198435281b4cc58215c8200, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/156d81d22610441da8a81c6c41032de7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/24910344e4cf43a387315884c56f2a0e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/fd711c181f044b5c9ba877029806d730, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4edd99b9862b4dc095468ee5725df007] to archive 2024-11-09T22:40:11,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:40:11,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e9e167697c7b424e930e1d48813999fa to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e9e167697c7b424e930e1d48813999fa 2024-11-09T22:40:11,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bcc626afa89d4b6f8ab27178596b4a06 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bcc626afa89d4b6f8ab27178596b4a06 2024-11-09T22:40:11,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/f7735160aae0429d966813e149a49a23 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/f7735160aae0429d966813e149a49a23 2024-11-09T22:40:11,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/dea556156bca4db6a597f80c6d023cdd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/dea556156bca4db6a597f80c6d023cdd 2024-11-09T22:40:11,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1bb4ebcc5d4d4130bf7eeca69fae1222 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1bb4ebcc5d4d4130bf7eeca69fae1222 2024-11-09T22:40:11,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7bb1c56391064abe9a7c06489263a6d3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7bb1c56391064abe9a7c06489263a6d3 2024-11-09T22:40:11,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/2b77b618f1a24065bbdf6abc8f7a206f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/2b77b618f1a24065bbdf6abc8f7a206f 2024-11-09T22:40:11,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1ac0311c59f2484ea2b88fe1a6611088 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/1ac0311c59f2484ea2b88fe1a6611088 2024-11-09T22:40:11,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4049b983abba4541849d882fe344ee92 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4049b983abba4541849d882fe344ee92 2024-11-09T22:40:11,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/9066252f5a56456a82a2079ac6dde416 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/9066252f5a56456a82a2079ac6dde416 2024-11-09T22:40:11,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/c5d1203ade834f7c8a6293167a3ac983 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/c5d1203ade834f7c8a6293167a3ac983 2024-11-09T22:40:11,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/a1bb68767ba346699dbbbb3d08dd5dfe to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/a1bb68767ba346699dbbbb3d08dd5dfe 2024-11-09T22:40:11,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4ef6297fe66d4b8b86f32586388627de to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4ef6297fe66d4b8b86f32586388627de 2024-11-09T22:40:11,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/073194f889dd46eabdc979b4eb3597ff to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/073194f889dd46eabdc979b4eb3597ff 2024-11-09T22:40:11,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6103b0172dc4470f973f529c76fda9f3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6103b0172dc4470f973f529c76fda9f3 2024-11-09T22:40:11,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8c948f48318f496289a0a1c3a2bc056a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8c948f48318f496289a0a1c3a2bc056a 2024-11-09T22:40:11,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6179c18658a34d759d9101e00293d97d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/6179c18658a34d759d9101e00293d97d 2024-11-09T22:40:11,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/93d23f31e1b6408581054dd26f4ac977 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/93d23f31e1b6408581054dd26f4ac977 2024-11-09T22:40:11,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/83856ef206f4421b9a5ee6824c7663af to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/83856ef206f4421b9a5ee6824c7663af 2024-11-09T22:40:11,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8acfa6c3fecd4aafba919e324cceb2d9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/8acfa6c3fecd4aafba919e324cceb2d9 2024-11-09T22:40:11,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e08ac95500464889867465ec483160bc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/e08ac95500464889867465ec483160bc 2024-11-09T22:40:11,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/37b3b8e6c68d44f98fcd009566236264 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/37b3b8e6c68d44f98fcd009566236264 2024-11-09T22:40:11,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/520f269af7614adcb28fe54294cde0b3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/520f269af7614adcb28fe54294cde0b3 2024-11-09T22:40:11,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/cae4490519b049248689c01084e8dadf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/cae4490519b049248689c01084e8dadf 2024-11-09T22:40:11,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/52e609b88b454b4d9342682e1c05d9cb to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/52e609b88b454b4d9342682e1c05d9cb 2024-11-09T22:40:11,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/de298dfe50b641c6b2a5f5f3c49f9fe8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/de298dfe50b641c6b2a5f5f3c49f9fe8 2024-11-09T22:40:11,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bbc2d9aa07aa4535a23872dff9f9e658 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/bbc2d9aa07aa4535a23872dff9f9e658 2024-11-09T22:40:11,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/02406a4df921482a86e0cd1b3e90c5e6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/02406a4df921482a86e0cd1b3e90c5e6 2024-11-09T22:40:11,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/69af8a046198435281b4cc58215c8200 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/69af8a046198435281b4cc58215c8200 2024-11-09T22:40:11,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/156d81d22610441da8a81c6c41032de7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/156d81d22610441da8a81c6c41032de7 2024-11-09T22:40:11,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/24910344e4cf43a387315884c56f2a0e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/24910344e4cf43a387315884c56f2a0e 2024-11-09T22:40:11,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/fd711c181f044b5c9ba877029806d730 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/fd711c181f044b5c9ba877029806d730 2024-11-09T22:40:11,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4edd99b9862b4dc095468ee5725df007 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/4edd99b9862b4dc095468ee5725df007 2024-11-09T22:40:11,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bdd93c98994a4a74867ad5a203c49fa3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/b488e71f9fee4631a60e7477bd185b97, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/5ce4d9f109d04758b7f17a8462db7a90, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ffb15218afb643248da38279bae7030c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8a4a5499d6194c6fa10663604f873dba, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/80b3efe5424e42e79c89a63f19a253f8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/1c0db947758f4b9f9c056b7c5206b161, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/018f624986e8422888f37cae713a1ef8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9a7be23ec5f34c9d9103ea6e7f282101, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/0589debe75004d9ea1001118388e0491, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/f46ccec201a54a3d85c3585cf04555d6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4bb6d0585aad40b6a5f739b4668dfaae, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/cd54e7a96f844bd6a1db7af8c9f01dc4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ca81ee443eeb4e849235b0691e85272d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e09a9b2f0f25449f8cbbf9d80c58b6b2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9801ca83991b49a79c1c5c86808a9068, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ab9caf0bf062491e87b3184fc3ff2211, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8e83423c8643458f98831b0c36635012, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/529f16694e9b4d958836f1351215da39, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bc0e81359939422fa20b9ab2bd95907f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/c1b77bae244d471f93127d4698bc8fd9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/651a3d08b879457abee9e7cf3645db71, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2cd6ee0e9c147e6918d37c1c198c36d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bebb6e062ad140ce9c3659a67cf744ed, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4a81a5e8542a4cf394863c68d06a3ad1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4b8cd0e5d138450a955bd84d4f96a62d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2a3e020421d4efd9d89d84a7ae0846e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3fa808ee68b343b2a85eb0d916ec0b56, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/10176ed925754bf09f4ffca6d31dfff2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8702c97f5d3c4a01ae1cd67c90a943bc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/fb7e60d6507d4ee688c2c14b91b17ec3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e361e2ab494f4f18bc132b703d34edbe, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e6b09c84e052464fab0de1d6eef0738a] to archive 2024-11-09T22:40:11,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:40:11,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bdd93c98994a4a74867ad5a203c49fa3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bdd93c98994a4a74867ad5a203c49fa3 2024-11-09T22:40:11,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/b488e71f9fee4631a60e7477bd185b97 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/b488e71f9fee4631a60e7477bd185b97 2024-11-09T22:40:11,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/5ce4d9f109d04758b7f17a8462db7a90 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/5ce4d9f109d04758b7f17a8462db7a90 2024-11-09T22:40:11,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ffb15218afb643248da38279bae7030c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ffb15218afb643248da38279bae7030c 2024-11-09T22:40:11,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8a4a5499d6194c6fa10663604f873dba to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8a4a5499d6194c6fa10663604f873dba 2024-11-09T22:40:11,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/80b3efe5424e42e79c89a63f19a253f8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/80b3efe5424e42e79c89a63f19a253f8 2024-11-09T22:40:11,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/1c0db947758f4b9f9c056b7c5206b161 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/1c0db947758f4b9f9c056b7c5206b161 2024-11-09T22:40:11,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/018f624986e8422888f37cae713a1ef8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/018f624986e8422888f37cae713a1ef8 2024-11-09T22:40:11,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9a7be23ec5f34c9d9103ea6e7f282101 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9a7be23ec5f34c9d9103ea6e7f282101 2024-11-09T22:40:11,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/0589debe75004d9ea1001118388e0491 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/0589debe75004d9ea1001118388e0491 2024-11-09T22:40:11,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/f46ccec201a54a3d85c3585cf04555d6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/f46ccec201a54a3d85c3585cf04555d6 2024-11-09T22:40:11,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4bb6d0585aad40b6a5f739b4668dfaae to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4bb6d0585aad40b6a5f739b4668dfaae 2024-11-09T22:40:11,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/cd54e7a96f844bd6a1db7af8c9f01dc4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/cd54e7a96f844bd6a1db7af8c9f01dc4 2024-11-09T22:40:11,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ca81ee443eeb4e849235b0691e85272d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ca81ee443eeb4e849235b0691e85272d 2024-11-09T22:40:11,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e09a9b2f0f25449f8cbbf9d80c58b6b2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e09a9b2f0f25449f8cbbf9d80c58b6b2 2024-11-09T22:40:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9801ca83991b49a79c1c5c86808a9068 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/9801ca83991b49a79c1c5c86808a9068 2024-11-09T22:40:11,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ab9caf0bf062491e87b3184fc3ff2211 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/ab9caf0bf062491e87b3184fc3ff2211 2024-11-09T22:40:11,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8e83423c8643458f98831b0c36635012 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8e83423c8643458f98831b0c36635012 2024-11-09T22:40:11,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/529f16694e9b4d958836f1351215da39 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/529f16694e9b4d958836f1351215da39 2024-11-09T22:40:11,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bc0e81359939422fa20b9ab2bd95907f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bc0e81359939422fa20b9ab2bd95907f 2024-11-09T22:40:11,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/c1b77bae244d471f93127d4698bc8fd9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/c1b77bae244d471f93127d4698bc8fd9 2024-11-09T22:40:11,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/651a3d08b879457abee9e7cf3645db71 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/651a3d08b879457abee9e7cf3645db71 2024-11-09T22:40:11,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2cd6ee0e9c147e6918d37c1c198c36d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2cd6ee0e9c147e6918d37c1c198c36d 2024-11-09T22:40:11,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bebb6e062ad140ce9c3659a67cf744ed to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/bebb6e062ad140ce9c3659a67cf744ed 2024-11-09T22:40:11,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4a81a5e8542a4cf394863c68d06a3ad1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4a81a5e8542a4cf394863c68d06a3ad1 2024-11-09T22:40:11,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4b8cd0e5d138450a955bd84d4f96a62d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/4b8cd0e5d138450a955bd84d4f96a62d 2024-11-09T22:40:11,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2a3e020421d4efd9d89d84a7ae0846e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d2a3e020421d4efd9d89d84a7ae0846e 2024-11-09T22:40:11,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3fa808ee68b343b2a85eb0d916ec0b56 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3fa808ee68b343b2a85eb0d916ec0b56 2024-11-09T22:40:11,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/10176ed925754bf09f4ffca6d31dfff2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/10176ed925754bf09f4ffca6d31dfff2 2024-11-09T22:40:11,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8702c97f5d3c4a01ae1cd67c90a943bc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/8702c97f5d3c4a01ae1cd67c90a943bc 2024-11-09T22:40:11,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/fb7e60d6507d4ee688c2c14b91b17ec3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/fb7e60d6507d4ee688c2c14b91b17ec3 2024-11-09T22:40:11,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e361e2ab494f4f18bc132b703d34edbe to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e361e2ab494f4f18bc132b703d34edbe 2024-11-09T22:40:11,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e6b09c84e052464fab0de1d6eef0738a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/e6b09c84e052464fab0de1d6eef0738a 2024-11-09T22:40:11,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4c23aa3ca0ba4734a308b18e802abac0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/30aea86b93d242da8113e3635de206e1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/578629df320546139ea408ecb6625da9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/90bf07e5e5d34a2b957f0ccf67991cd1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c9af2d14cc844f60a0a048b1b66bcea1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/d5363b2f9ccb4e7a86d161556562b6d5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/20fa1e3f6ca048f291d1c343ef8e00be, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cafd707a6a2e488da0f4e2b222f8a244, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/ed09de8316eb4dcc819afa839ac4f567, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/01ceb183123142a2acf84b72cce90b44, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/394b245be35b456a827548d867eccb6e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/de54bc44d2fc4f1ba92f28310380d394, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c0c2ccd1d3d64d06a7be4e48d985cfdc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/9b0293f4617f4ec3a71f0298edc8eb16, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/f9a4b64aa163403ea24db4aa62abf6c2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/99fb485527014dc099c86b609b70ebdc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cf362435bdd04b1788508e4424bcc302, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/e28a1fb7dbc542529e59e711ddf7c62d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5707eb32a1bd46398be9e6d68e6b367e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4cec082d8fe0450c9c608322e2c04b3a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5c3b554bbc1a40d58226aac05b75f031, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/6abb3e6650e740119dd5fa90c9469a82, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/2bdb018edbba4631ae6259bfb094497a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/3ff34409363f4d8480aa907388f4dcd9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/504037c5634c416e8b0a66082b5e2f5f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/be44e331c9fb4f0f840b99df0b0a19f5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/21530fa6541046cdadb9881092392ba0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86837fbc46c143b9881a05f3ad539b4a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/66cb2c6741304bf1a9a39e36b16b7ebf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/b8dd65a3360045a4b58665e426430408, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/8d7ccf1be5f744bc8c231941d666cb0f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/772b11179a594469b728e999fc1c8cb2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/55d65950332343d0a2801d5c029bccb9] to archive 2024-11-09T22:40:11,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:40:11,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4c23aa3ca0ba4734a308b18e802abac0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4c23aa3ca0ba4734a308b18e802abac0 2024-11-09T22:40:11,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/30aea86b93d242da8113e3635de206e1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/30aea86b93d242da8113e3635de206e1 2024-11-09T22:40:11,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/578629df320546139ea408ecb6625da9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/578629df320546139ea408ecb6625da9 2024-11-09T22:40:11,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/90bf07e5e5d34a2b957f0ccf67991cd1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/90bf07e5e5d34a2b957f0ccf67991cd1 2024-11-09T22:40:11,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c9af2d14cc844f60a0a048b1b66bcea1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c9af2d14cc844f60a0a048b1b66bcea1 2024-11-09T22:40:11,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/d5363b2f9ccb4e7a86d161556562b6d5 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/d5363b2f9ccb4e7a86d161556562b6d5 2024-11-09T22:40:11,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/20fa1e3f6ca048f291d1c343ef8e00be to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/20fa1e3f6ca048f291d1c343ef8e00be 2024-11-09T22:40:11,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cafd707a6a2e488da0f4e2b222f8a244 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cafd707a6a2e488da0f4e2b222f8a244 2024-11-09T22:40:11,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/ed09de8316eb4dcc819afa839ac4f567 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/ed09de8316eb4dcc819afa839ac4f567 2024-11-09T22:40:11,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/01ceb183123142a2acf84b72cce90b44 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/01ceb183123142a2acf84b72cce90b44 2024-11-09T22:40:11,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/394b245be35b456a827548d867eccb6e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/394b245be35b456a827548d867eccb6e 2024-11-09T22:40:11,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/de54bc44d2fc4f1ba92f28310380d394 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/de54bc44d2fc4f1ba92f28310380d394 2024-11-09T22:40:11,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c0c2ccd1d3d64d06a7be4e48d985cfdc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c0c2ccd1d3d64d06a7be4e48d985cfdc 2024-11-09T22:40:11,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/9b0293f4617f4ec3a71f0298edc8eb16 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/9b0293f4617f4ec3a71f0298edc8eb16 2024-11-09T22:40:11,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/f9a4b64aa163403ea24db4aa62abf6c2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/f9a4b64aa163403ea24db4aa62abf6c2 2024-11-09T22:40:11,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/99fb485527014dc099c86b609b70ebdc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/99fb485527014dc099c86b609b70ebdc 2024-11-09T22:40:11,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cf362435bdd04b1788508e4424bcc302 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/cf362435bdd04b1788508e4424bcc302 2024-11-09T22:40:11,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/e28a1fb7dbc542529e59e711ddf7c62d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/e28a1fb7dbc542529e59e711ddf7c62d 2024-11-09T22:40:11,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5707eb32a1bd46398be9e6d68e6b367e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5707eb32a1bd46398be9e6d68e6b367e 2024-11-09T22:40:11,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4cec082d8fe0450c9c608322e2c04b3a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/4cec082d8fe0450c9c608322e2c04b3a 2024-11-09T22:40:11,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5c3b554bbc1a40d58226aac05b75f031 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/5c3b554bbc1a40d58226aac05b75f031 2024-11-09T22:40:11,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/6abb3e6650e740119dd5fa90c9469a82 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/6abb3e6650e740119dd5fa90c9469a82 2024-11-09T22:40:11,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/2bdb018edbba4631ae6259bfb094497a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/2bdb018edbba4631ae6259bfb094497a 2024-11-09T22:40:11,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/3ff34409363f4d8480aa907388f4dcd9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/3ff34409363f4d8480aa907388f4dcd9 2024-11-09T22:40:11,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/504037c5634c416e8b0a66082b5e2f5f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/504037c5634c416e8b0a66082b5e2f5f 2024-11-09T22:40:11,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/be44e331c9fb4f0f840b99df0b0a19f5 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/be44e331c9fb4f0f840b99df0b0a19f5 2024-11-09T22:40:11,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/21530fa6541046cdadb9881092392ba0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/21530fa6541046cdadb9881092392ba0 2024-11-09T22:40:11,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86837fbc46c143b9881a05f3ad539b4a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86837fbc46c143b9881a05f3ad539b4a 2024-11-09T22:40:11,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/66cb2c6741304bf1a9a39e36b16b7ebf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/66cb2c6741304bf1a9a39e36b16b7ebf 2024-11-09T22:40:11,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/b8dd65a3360045a4b58665e426430408 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/b8dd65a3360045a4b58665e426430408 2024-11-09T22:40:11,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/8d7ccf1be5f744bc8c231941d666cb0f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/8d7ccf1be5f744bc8c231941d666cb0f 2024-11-09T22:40:11,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/772b11179a594469b728e999fc1c8cb2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/772b11179a594469b728e999fc1c8cb2 2024-11-09T22:40:11,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/55d65950332343d0a2801d5c029bccb9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/55d65950332343d0a2801d5c029bccb9 2024-11-09T22:40:11,199 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/recovered.edits/512.seqid, newMaxSeqId=512, maxSeqId=1 2024-11-09T22:40:11,199 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. 2024-11-09T22:40:11,200 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 1f18320792a575fd182c9198939e0c16: Waiting for close lock at 1731192009440Running coprocessor pre-close hooks at 1731192009440Disabling compacts and flushes for region at 1731192009440Disabling writes for close at 1731192009786 (+346 ms)Obtaining lock to block concurrent updates at 1731192009786Preparing flush snapshotting stores in 1f18320792a575fd182c9198939e0c16 at 1731192009786Finished memstore snapshotting TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731192009787 (+1 ms)Flushing stores of TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16. at 1731192009787Flushing 1f18320792a575fd182c9198939e0c16/A: creating writer at 1731192009787Flushing 1f18320792a575fd182c9198939e0c16/A: appending metadata at 1731192009790 (+3 ms)Flushing 1f18320792a575fd182c9198939e0c16/A: closing flushed file at 1731192009790Flushing 1f18320792a575fd182c9198939e0c16/B: creating writer at 1731192010205 (+415 ms)Flushing 1f18320792a575fd182c9198939e0c16/B: appending metadata at 1731192010210 (+5 ms)Flushing 1f18320792a575fd182c9198939e0c16/B: closing flushed file at 1731192010210Flushing 1f18320792a575fd182c9198939e0c16/C: creating writer at 1731192010621 (+411 ms)Flushing 1f18320792a575fd182c9198939e0c16/C: appending metadata at 1731192010624 (+3 ms)Flushing 1f18320792a575fd182c9198939e0c16/C: closing flushed file at 1731192010624Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5692b642: reopening flushed file at 1731192011032 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79e3eb90: reopening flushed file at 1731192011039 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47e3a9e0: reopening flushed file at 1731192011045 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1f18320792a575fd182c9198939e0c16 in 1266ms, sequenceid=509, compaction requested=true at 1731192011053 (+8 ms)Writing region close event to WAL at 1731192011195 (+142 ms)Running coprocessor post-close hooks at 1731192011199 (+4 ms)Closed at 1731192011199 2024-11-09T22:40:11,202 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:11,203 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1f18320792a575fd182c9198939e0c16, regionState=CLOSED 2024-11-09T22:40:11,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:40:11,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T22:40:11,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 1f18320792a575fd182c9198939e0c16, server=9d8c37854f6e,35101,1731191981027 in 1.9260 sec 2024-11-09T22:40:11,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-09T22:40:11,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f18320792a575fd182c9198939e0c16, UNASSIGN in 1.9320 sec 2024-11-09T22:40:11,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-09T22:40:11,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9380 sec 2024-11-09T22:40:11,212 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192011212"}]},"ts":"1731192011212"} 2024-11-09T22:40:11,214 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T22:40:11,214 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T22:40:11,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9740 sec 2024-11-09T22:40:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-09T22:40:11,416 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:11,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T22:40:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:11,431 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-09T22:40:11,432 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:11,439 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:11,444 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/recovered.edits] 2024-11-09T22:40:11,447 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7802a57df3184bd8830b44cc47012a70 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7802a57df3184bd8830b44cc47012a70 2024-11-09T22:40:11,449 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7f88000d0755416f9f636e5986d5c83a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/7f88000d0755416f9f636e5986d5c83a 2024-11-09T22:40:11,450 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/927e3ad7210941f5a4eb7a205f18aa9f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/927e3ad7210941f5a4eb7a205f18aa9f 2024-11-09T22:40:11,451 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/afacd2aa984840e8ace4b94a84938732 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/A/afacd2aa984840e8ace4b94a84938732 2024-11-09T22:40:11,453 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/256c1f7e6bf74d2cbfda28f4c4550508 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/256c1f7e6bf74d2cbfda28f4c4550508 2024-11-09T22:40:11,455 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3b6b9e00f59d4c84aeffeb991574a285 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/3b6b9e00f59d4c84aeffeb991574a285 2024-11-09T22:40:11,456 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/7f1191fd60a2485995e0ae02f0cb40bc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/7f1191fd60a2485995e0ae02f0cb40bc 2024-11-09T22:40:11,457 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d5849d9b38b84d8cbcac1dc7278b2d01 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/B/d5849d9b38b84d8cbcac1dc7278b2d01 2024-11-09T22:40:11,459 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/83dcda76afbe4bd5bea87a14fd58f6a8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/83dcda76afbe4bd5bea87a14fd58f6a8 2024-11-09T22:40:11,460 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86303de5f5454da5ad8a98838b4e834e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/86303de5f5454da5ad8a98838b4e834e 2024-11-09T22:40:11,461 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c478918045e44d45987743bcae2cafee to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/c478918045e44d45987743bcae2cafee 2024-11-09T22:40:11,462 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/eef75837d205487ab54fea046a10dfee to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/C/eef75837d205487ab54fea046a10dfee 2024-11-09T22:40:11,465 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/recovered.edits/512.seqid to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16/recovered.edits/512.seqid 2024-11-09T22:40:11,466 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1f18320792a575fd182c9198939e0c16 2024-11-09T22:40:11,466 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T22:40:11,471 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:11,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-09T22:40:11,482 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T22:40:11,510 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T22:40:11,512 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:11,512 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T22:40:11,512 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731192011512"}]},"ts":"9223372036854775807"} 2024-11-09T22:40:11,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T22:40:11,516 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1f18320792a575fd182c9198939e0c16, NAME => 'TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T22:40:11,516 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T22:40:11,516 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731192011516"}]},"ts":"9223372036854775807"} 2024-11-09T22:40:11,519 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T22:40:11,520 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:11,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 97 msec 2024-11-09T22:40:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-09T22:40:11,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T22:40:11,535 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:11,548 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1065910943_22 at /127.0.0.1:49366 [Waiting for operation #250] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9d8c37854f6e:35101-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1065910943_22 at /127.0.0.1:55344 [Waiting for operation #269] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-870303366_22 at /127.0.0.1:49580 [Waiting for operation #177] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=499 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17992 (was 19016) 2024-11-09T22:40:11,557 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=499, ProcessCount=11, AvailableMemoryMB=17992 2024-11-09T22:40:11,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:40:11,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:40:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:11,561 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T22:40:11,561 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:11,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-09T22:40:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T22:40:11,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T22:40:11,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741948_1124 (size=959) 2024-11-09T22:40:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T22:40:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T22:40:11,971 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 2024-11-09T22:40:11,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741949_1125 (size=53) 2024-11-09T22:40:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T22:40:12,384 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:40:12,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e7b04b07f99af4a950a5462bdc4e6dc3, disabling compactions & flushes 2024-11-09T22:40:12,385 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:12,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:12,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. after waiting 0 ms 2024-11-09T22:40:12,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:12,385 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:12,386 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7b04b07f99af4a950a5462bdc4e6dc3: Waiting for close lock at 1731192012385Disabling compacts and flushes for region at 1731192012385Disabling writes for close at 1731192012385Writing region close event to WAL at 1731192012385Closed at 1731192012385 2024-11-09T22:40:12,389 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T22:40:12,389 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731192012389"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731192012389"}]},"ts":"1731192012389"} 2024-11-09T22:40:12,392 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T22:40:12,393 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T22:40:12,394 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192012393"}]},"ts":"1731192012393"} 2024-11-09T22:40:12,395 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T22:40:12,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, ASSIGN}] 2024-11-09T22:40:12,397 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, ASSIGN 2024-11-09T22:40:12,397 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, ASSIGN; state=OFFLINE, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=false 2024-11-09T22:40:12,548 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e7b04b07f99af4a950a5462bdc4e6dc3, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:12,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, ASSIGN because future has completed 2024-11-09T22:40:12,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:40:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T22:40:12,714 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:12,714 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:40:12,715 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,715 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:40:12,716 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,716 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,717 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,719 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7b04b07f99af4a950a5462bdc4e6dc3 columnFamilyName A 2024-11-09T22:40:12,720 DEBUG [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:12,720 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(327): Store=e7b04b07f99af4a950a5462bdc4e6dc3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:12,720 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,721 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7b04b07f99af4a950a5462bdc4e6dc3 columnFamilyName B 2024-11-09T22:40:12,721 DEBUG [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:12,722 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(327): Store=e7b04b07f99af4a950a5462bdc4e6dc3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:12,722 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,723 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7b04b07f99af4a950a5462bdc4e6dc3 columnFamilyName C 2024-11-09T22:40:12,723 DEBUG [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:12,723 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(327): Store=e7b04b07f99af4a950a5462bdc4e6dc3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:12,724 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,724 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,724 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,725 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,725 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,726 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:40:12,727 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,729 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:40:12,729 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened e7b04b07f99af4a950a5462bdc4e6dc3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68540538, jitterRate=0.02133360505104065}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:40:12,729 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:12,730 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for e7b04b07f99af4a950a5462bdc4e6dc3: Running coprocessor pre-open hook at 1731192012716Writing region info on filesystem at 1731192012716Initializing all the Stores at 1731192012717 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192012717Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192012717Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192012717Cleaning up temporary data from old regions at 1731192012725 (+8 ms)Running coprocessor post-open hooks at 1731192012729 (+4 ms)Region opened successfully at 1731192012730 (+1 ms) 2024-11-09T22:40:12,731 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., pid=36, masterSystemTime=1731192012705 2024-11-09T22:40:12,733 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:12,733 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:12,734 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e7b04b07f99af4a950a5462bdc4e6dc3, regionState=OPEN, openSeqNum=2, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:12,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:40:12,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-09T22:40:12,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 in 186 msec 2024-11-09T22:40:12,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-09T22:40:12,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, ASSIGN in 344 msec 2024-11-09T22:40:12,743 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T22:40:12,743 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192012743"}]},"ts":"1731192012743"} 2024-11-09T22:40:12,745 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T22:40:12,746 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T22:40:12,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-09T22:40:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-09T22:40:13,705 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:13,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:13,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:13,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:13,708 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336396c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:13,710 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37421322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:13,710 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:13,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:13,714 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T22:40:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d8c37854f6e,34651,1731191980160 2024-11-09T22:40:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3df6455a 2024-11-09T22:40:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T22:40:13,716 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T22:40:13,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:40:13,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:40:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:13,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741950_1126 (size=995) 2024-11-09T22:40:14,147 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T22:40:14,147 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T22:40:14,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:40:14,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, REOPEN/MOVE}] 2024-11-09T22:40:14,162 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, REOPEN/MOVE 2024-11-09T22:40:14,163 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e7b04b07f99af4a950a5462bdc4e6dc3, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:14,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, REOPEN/MOVE because future has completed 2024-11-09T22:40:14,165 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:40:14,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:40:14,319 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,319 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:40:14,319 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing e7b04b07f99af4a950a5462bdc4e6dc3, disabling compactions & flushes 2024-11-09T22:40:14,319 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:14,319 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:14,319 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. after waiting 0 ms 2024-11-09T22:40:14,319 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:14,323 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T22:40:14,324 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:14,324 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for e7b04b07f99af4a950a5462bdc4e6dc3: Waiting for close lock at 1731192014319Running coprocessor pre-close hooks at 1731192014319Disabling compacts and flushes for region at 1731192014319Disabling writes for close at 1731192014319Writing region close event to WAL at 1731192014320 (+1 ms)Running coprocessor post-close hooks at 1731192014324 (+4 ms)Closed at 1731192014324 2024-11-09T22:40:14,324 WARN [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: e7b04b07f99af4a950a5462bdc4e6dc3 to self. 2024-11-09T22:40:14,326 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e7b04b07f99af4a950a5462bdc4e6dc3, regionState=CLOSED 2024-11-09T22:40:14,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:40:14,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-09T22:40:14,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 in 165 msec 2024-11-09T22:40:14,333 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, REOPEN/MOVE; state=CLOSED, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=true 2024-11-09T22:40:14,483 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e7b04b07f99af4a950a5462bdc4e6dc3, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:14,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, REOPEN/MOVE because future has completed 2024-11-09T22:40:14,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:40:14,642 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:14,642 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:40:14,643 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,643 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:40:14,643 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,643 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,648 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,655 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7b04b07f99af4a950a5462bdc4e6dc3 columnFamilyName A 2024-11-09T22:40:14,658 DEBUG [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:14,658 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(327): Store=e7b04b07f99af4a950a5462bdc4e6dc3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:14,659 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,660 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7b04b07f99af4a950a5462bdc4e6dc3 columnFamilyName B 2024-11-09T22:40:14,660 DEBUG [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:14,661 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(327): Store=e7b04b07f99af4a950a5462bdc4e6dc3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:14,661 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,662 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7b04b07f99af4a950a5462bdc4e6dc3 columnFamilyName C 2024-11-09T22:40:14,662 DEBUG [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:14,663 INFO [StoreOpener-e7b04b07f99af4a950a5462bdc4e6dc3-1 {}] regionserver.HStore(327): Store=e7b04b07f99af4a950a5462bdc4e6dc3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:14,663 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,664 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,665 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,666 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,666 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,667 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:40:14,669 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,670 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened e7b04b07f99af4a950a5462bdc4e6dc3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75143931, jitterRate=0.1197318285703659}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:40:14,670 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,671 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for e7b04b07f99af4a950a5462bdc4e6dc3: Running coprocessor pre-open hook at 1731192014643Writing region info on filesystem at 1731192014643Initializing all the Stores at 1731192014644 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192014644Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192014644Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192014644Cleaning up temporary data from old regions at 1731192014666 (+22 ms)Running coprocessor post-open hooks at 1731192014670 (+4 ms)Region opened successfully at 1731192014671 (+1 ms) 2024-11-09T22:40:14,672 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., pid=41, masterSystemTime=1731192014638 2024-11-09T22:40:14,675 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:14,675 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:14,675 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e7b04b07f99af4a950a5462bdc4e6dc3, regionState=OPEN, openSeqNum=5, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:14,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:40:14,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-09T22:40:14,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 in 193 msec 2024-11-09T22:40:14,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-09T22:40:14,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, REOPEN/MOVE in 522 msec 2024-11-09T22:40:14,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-09T22:40:14,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-11-09T22:40:14,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 972 msec 2024-11-09T22:40:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-09T22:40:14,700 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:14,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2c1aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,705 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,708 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1389b68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,711 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a16c961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459f4ecc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,716 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c79c95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,719 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21df1cd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,721 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4a9804, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,728 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32a52868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,731 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459be2bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,736 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29e0f7d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,739 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395f6c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754865c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,745 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd0e0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,753 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90d2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,755 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1701504c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:14,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:14,760 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29de1d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:14,762 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:14,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:14,770 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,772 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,772 DEBUG [Thread-633 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,773 DEBUG [Thread-633 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,775 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,778 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,779 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,780 DEBUG [Thread-648 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,780 DEBUG [Thread-648 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,781 DEBUG [Thread-644 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,784 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:14,804 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:14,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:40:14,820 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,822 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,828 DEBUG [Thread-639 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,830 DEBUG [Thread-639 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:14,841 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-09T22:40:14,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:14,844 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:14,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T22:40:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:14,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:40:14,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ced95a9e7c144cf498c9d671af29ce50_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_1/A:col10/1731192014802/Put/seqid=0 2024-11-09T22:40:14,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192074907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:14,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192074911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:14,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192074911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192074913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192074913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:14,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741951_1127 (size=12154) 2024-11-09T22:40:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T22:40:15,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192075025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192075036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192075036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192075036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192075036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T22:40:15,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192075245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192075255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192075255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192075255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192075256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:15,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:15,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,337 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:15,343 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ced95a9e7c144cf498c9d671af29ce50_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ced95a9e7c144cf498c9d671af29ce50_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:15,351 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/b972422bfd0b447596122ecd07b60389, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:15,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/b972422bfd0b447596122ecd07b60389 is 175, key is test_row_1/A:col10/1731192014802/Put/seqid=0 2024-11-09T22:40:15,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741952_1128 (size=30951) 2024-11-09T22:40:15,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T22:40:15,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192075555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192075565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192075565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192075565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192075569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:15,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,796 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/b972422bfd0b447596122ecd07b60389 2024-11-09T22:40:15,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/8383e82c1ee345ca92254592e0f06764 is 50, key is test_row_1/B:col10/1731192014802/Put/seqid=0 2024-11-09T22:40:15,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741953_1129 (size=9657) 2024-11-09T22:40:15,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/8383e82c1ee345ca92254592e0f06764 2024-11-09T22:40:15,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/59de11c459f84020ace4c22fb0e244f9 is 50, key is test_row_1/C:col10/1731192014802/Put/seqid=0 2024-11-09T22:40:15,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:15,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:15,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:15,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:15,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741954_1130 (size=9657) 2024-11-09T22:40:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T22:40:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192076065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192076075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192076075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192076075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:16,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:16,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192076086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:16,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:16,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:16,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:16,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:16,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/59de11c459f84020ace4c22fb0e244f9 2024-11-09T22:40:16,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/b972422bfd0b447596122ecd07b60389 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b972422bfd0b447596122ecd07b60389 2024-11-09T22:40:16,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b972422bfd0b447596122ecd07b60389, entries=150, sequenceid=16, filesize=30.2 K 2024-11-09T22:40:16,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/8383e82c1ee345ca92254592e0f06764 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/8383e82c1ee345ca92254592e0f06764 2024-11-09T22:40:16,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:16,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:16,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:16,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/8383e82c1ee345ca92254592e0f06764, entries=100, sequenceid=16, filesize=9.4 K 2024-11-09T22:40:16,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/59de11c459f84020ace4c22fb0e244f9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/59de11c459f84020ace4c22fb0e244f9 2024-11-09T22:40:16,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/59de11c459f84020ace4c22fb0e244f9, entries=100, sequenceid=16, filesize=9.4 K 2024-11-09T22:40:16,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1618ms, sequenceid=16, compaction requested=false 2024-11-09T22:40:16,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:16,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T22:40:16,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:16,554 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:40:16,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a1976dc1a3c64d8bbee9f74ae3af7fda_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192014897/Put/seqid=0 2024-11-09T22:40:16,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741955_1131 (size=12154) 2024-11-09T22:40:16,745 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T22:40:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T22:40:17,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:17,030 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a1976dc1a3c64d8bbee9f74ae3af7fda_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a1976dc1a3c64d8bbee9f74ae3af7fda_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/ed1d91fb432144fa93a6c53c0c08f647, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:17,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/ed1d91fb432144fa93a6c53c0c08f647 is 175, key is test_row_0/A:col10/1731192014897/Put/seqid=0 2024-11-09T22:40:17,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741956_1132 (size=30955) 2024-11-09T22:40:17,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:17,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192077105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,111 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in e7b04b07f99af4a950a5462bdc4e6dc3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:40:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192077109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192077111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192077114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192077109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192077225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192077225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192077226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192077235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192077446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,449 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/ed1d91fb432144fa93a6c53c0c08f647 2024-11-09T22:40:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192077446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192077446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192077460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/9549263903694d74903a52c7efee0d82 is 50, key is test_row_0/B:col10/1731192014897/Put/seqid=0 2024-11-09T22:40:17,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741957_1133 (size=12001) 2024-11-09T22:40:17,543 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/9549263903694d74903a52c7efee0d82 2024-11-09T22:40:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/ce70e3acb5c745f2a325724bedbb9574 is 50, key is test_row_0/C:col10/1731192014897/Put/seqid=0 2024-11-09T22:40:17,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741958_1134 (size=12001) 2024-11-09T22:40:17,600 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/ce70e3acb5c745f2a325724bedbb9574 2024-11-09T22:40:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/ed1d91fb432144fa93a6c53c0c08f647 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed1d91fb432144fa93a6c53c0c08f647 2024-11-09T22:40:17,627 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed1d91fb432144fa93a6c53c0c08f647, entries=150, sequenceid=41, filesize=30.2 K 2024-11-09T22:40:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/9549263903694d74903a52c7efee0d82 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/9549263903694d74903a52c7efee0d82 2024-11-09T22:40:17,639 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/9549263903694d74903a52c7efee0d82, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T22:40:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/ce70e3acb5c745f2a325724bedbb9574 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/ce70e3acb5c745f2a325724bedbb9574 2024-11-09T22:40:17,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/ce70e3acb5c745f2a325724bedbb9574, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T22:40:17,659 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1104ms, sequenceid=41, compaction requested=false 2024-11-09T22:40:17,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:17,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:17,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-09T22:40:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-09T22:40:17,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T22:40:17,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8170 sec 2024-11-09T22:40:17,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.8250 sec 2024-11-09T22:40:17,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:17,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T22:40:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:17,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ece38e63031c475e8125d324493766ba_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192017107/Put/seqid=0 2024-11-09T22:40:17,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741959_1135 (size=14594) 2024-11-09T22:40:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192077863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192077868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192077877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192077879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192077985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192077985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:17,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:17,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:17,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192077995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192077995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192078195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192078196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192078204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192078205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,237 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:18,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ece38e63031c475e8125d324493766ba_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ece38e63031c475e8125d324493766ba_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:18,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/67996d8361b545adb3f06d76d1601666, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:18,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/67996d8361b545adb3f06d76d1601666 is 175, key is test_row_0/A:col10/1731192017107/Put/seqid=0 2024-11-09T22:40:18,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741960_1136 (size=39549) 2024-11-09T22:40:18,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192078505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192078506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192078515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192078515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:18,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/67996d8361b545adb3f06d76d1601666 2024-11-09T22:40:18,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/665fcc7e05444ab1a3415c7b07e3e774 is 50, key is test_row_0/B:col10/1731192017107/Put/seqid=0 2024-11-09T22:40:18,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741961_1137 (size=12001) 2024-11-09T22:40:19,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192079015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:19,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192079015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T22:40:19,026 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:19,026 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-09T22:40:19,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192079025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192079026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T22:40:19,031 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:19,033 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:19,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:19,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/665fcc7e05444ab1a3415c7b07e3e774 2024-11-09T22:40:19,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/70bdafa821f14467ae7a58b69d9b3066 is 50, key is test_row_0/C:col10/1731192017107/Put/seqid=0 2024-11-09T22:40:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T22:40:19,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741962_1138 (size=12001) 2024-11-09T22:40:19,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/70bdafa821f14467ae7a58b69d9b3066 2024-11-09T22:40:19,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192079136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:19,142 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in e7b04b07f99af4a950a5462bdc4e6dc3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:40:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:19,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/67996d8361b545adb3f06d76d1601666 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/67996d8361b545adb3f06d76d1601666 2024-11-09T22:40:19,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/67996d8361b545adb3f06d76d1601666, entries=200, sequenceid=54, filesize=38.6 K 2024-11-09T22:40:19,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/665fcc7e05444ab1a3415c7b07e3e774 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/665fcc7e05444ab1a3415c7b07e3e774 2024-11-09T22:40:19,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/665fcc7e05444ab1a3415c7b07e3e774, entries=150, sequenceid=54, filesize=11.7 K 2024-11-09T22:40:19,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/70bdafa821f14467ae7a58b69d9b3066 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/70bdafa821f14467ae7a58b69d9b3066 2024-11-09T22:40:19,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/70bdafa821f14467ae7a58b69d9b3066, entries=150, sequenceid=54, filesize=11.7 K 2024-11-09T22:40:19,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1419ms, sequenceid=54, compaction requested=true 2024-11-09T22:40:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:19,178 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:19,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:19,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:19,179 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:19,180 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/A is initiating minor compaction (all files) 2024-11-09T22:40:19,180 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/A in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:19,180 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b972422bfd0b447596122ecd07b60389, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed1d91fb432144fa93a6c53c0c08f647, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/67996d8361b545adb3f06d76d1601666] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=99.1 K 2024-11-09T22:40:19,180 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:19,180 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b972422bfd0b447596122ecd07b60389, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed1d91fb432144fa93a6c53c0c08f647, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/67996d8361b545adb3f06d76d1601666] 2024-11-09T22:40:19,181 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:19,181 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b972422bfd0b447596122ecd07b60389, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731192014799 2024-11-09T22:40:19,181 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/B is initiating minor compaction (all files) 2024-11-09T22:40:19,181 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/B in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:19,181 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/8383e82c1ee345ca92254592e0f06764, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/9549263903694d74903a52c7efee0d82, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/665fcc7e05444ab1a3415c7b07e3e774] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=32.9 K 2024-11-09T22:40:19,182 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1d91fb432144fa93a6c53c0c08f647, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731192014897 2024-11-09T22:40:19,183 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67996d8361b545adb3f06d76d1601666, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731192017096 2024-11-09T22:40:19,183 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8383e82c1ee345ca92254592e0f06764, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731192014802 2024-11-09T22:40:19,183 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9549263903694d74903a52c7efee0d82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731192014897 2024-11-09T22:40:19,184 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 665fcc7e05444ab1a3415c7b07e3e774, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731192017096 2024-11-09T22:40:19,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T22:40:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:19,188 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T22:40:19,212 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:19,216 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:19,216 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/db4731e58cc648438a0420073c9ee3ac is 50, key is test_row_0/B:col10/1731192017107/Put/seqid=0 2024-11-09T22:40:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fe795347e5e24a9eb0af9f0581bc2700_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192017860/Put/seqid=0 2024-11-09T22:40:19,223 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110922a89bb6ca26442396c6a51ab658859c_e7b04b07f99af4a950a5462bdc4e6dc3 store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:19,229 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110922a89bb6ca26442396c6a51ab658859c_e7b04b07f99af4a950a5462bdc4e6dc3, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:19,230 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110922a89bb6ca26442396c6a51ab658859c_e7b04b07f99af4a950a5462bdc4e6dc3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:19,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741963_1139 (size=12104) 2024-11-09T22:40:19,274 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/db4731e58cc648438a0420073c9ee3ac as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/db4731e58cc648438a0420073c9ee3ac 2024-11-09T22:40:19,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741964_1140 (size=12154) 2024-11-09T22:40:19,289 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/B of e7b04b07f99af4a950a5462bdc4e6dc3 into db4731e58cc648438a0420073c9ee3ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:19,290 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:19,290 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/B, priority=13, startTime=1731192019178; duration=0sec 2024-11-09T22:40:19,290 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:19,290 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:B 2024-11-09T22:40:19,290 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:19,292 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:19,292 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/C is initiating minor compaction (all files) 2024-11-09T22:40:19,292 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/C in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:19,293 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/59de11c459f84020ace4c22fb0e244f9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/ce70e3acb5c745f2a325724bedbb9574, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/70bdafa821f14467ae7a58b69d9b3066] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=32.9 K 2024-11-09T22:40:19,293 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 59de11c459f84020ace4c22fb0e244f9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731192014802 2024-11-09T22:40:19,295 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ce70e3acb5c745f2a325724bedbb9574, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731192014897 2024-11-09T22:40:19,297 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 70bdafa821f14467ae7a58b69d9b3066, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731192017096 2024-11-09T22:40:19,306 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:19,306 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/c2dfa43e4d6047c586d0b4db696681ac is 50, key is test_row_0/C:col10/1731192017107/Put/seqid=0 2024-11-09T22:40:19,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741965_1141 (size=4469) 2024-11-09T22:40:19,323 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#A#compaction#120 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:19,324 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/103e9ac949624aa78281488b4a21322f is 175, key is test_row_0/A:col10/1731192017107/Put/seqid=0 2024-11-09T22:40:19,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741966_1142 (size=12104) 2024-11-09T22:40:19,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741967_1143 (size=31058) 2024-11-09T22:40:19,336 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/c2dfa43e4d6047c586d0b4db696681ac as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/c2dfa43e4d6047c586d0b4db696681ac 2024-11-09T22:40:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T22:40:19,346 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/C of e7b04b07f99af4a950a5462bdc4e6dc3 into c2dfa43e4d6047c586d0b4db696681ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:19,346 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:19,346 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/C, priority=13, startTime=1731192019178; duration=0sec 2024-11-09T22:40:19,346 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:19,346 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:C 2024-11-09T22:40:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T22:40:19,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,690 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fe795347e5e24a9eb0af9f0581bc2700_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fe795347e5e24a9eb0af9f0581bc2700_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/8d7b227fe37a47c8b946f3656746bbbd, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/8d7b227fe37a47c8b946f3656746bbbd is 175, key is test_row_0/A:col10/1731192017860/Put/seqid=0 2024-11-09T22:40:19,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741968_1144 (size=30955) 2024-11-09T22:40:19,698 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/8d7b227fe37a47c8b946f3656746bbbd 2024-11-09T22:40:19,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/60ef9d7a6b8241b4820f771a268e025d is 50, key is test_row_0/B:col10/1731192017860/Put/seqid=0 2024-11-09T22:40:19,747 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/103e9ac949624aa78281488b4a21322f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/103e9ac949624aa78281488b4a21322f 2024-11-09T22:40:19,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741969_1145 (size=12001) 2024-11-09T22:40:19,754 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/60ef9d7a6b8241b4820f771a268e025d 2024-11-09T22:40:19,765 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/A of e7b04b07f99af4a950a5462bdc4e6dc3 into 103e9ac949624aa78281488b4a21322f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:19,765 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:19,765 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/A, priority=13, startTime=1731192019178; duration=0sec 2024-11-09T22:40:19,765 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:19,765 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:A 2024-11-09T22:40:19,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/15b1590348e54407bc9ba4d9aa2ef7c6 is 50, key is test_row_0/C:col10/1731192017860/Put/seqid=0 2024-11-09T22:40:19,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741970_1146 (size=12001) 2024-11-09T22:40:19,831 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/15b1590348e54407bc9ba4d9aa2ef7c6 2024-11-09T22:40:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/8d7b227fe37a47c8b946f3656746bbbd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/8d7b227fe37a47c8b946f3656746bbbd 2024-11-09T22:40:19,845 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/8d7b227fe37a47c8b946f3656746bbbd, entries=150, sequenceid=78, filesize=30.2 K 2024-11-09T22:40:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/60ef9d7a6b8241b4820f771a268e025d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/60ef9d7a6b8241b4820f771a268e025d 2024-11-09T22:40:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,853 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/60ef9d7a6b8241b4820f771a268e025d, entries=150, sequenceid=78, filesize=11.7 K 2024-11-09T22:40:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/15b1590348e54407bc9ba4d9aa2ef7c6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/15b1590348e54407bc9ba4d9aa2ef7c6 2024-11-09T22:40:19,868 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/15b1590348e54407bc9ba4d9aa2ef7c6, entries=150, sequenceid=78, filesize=11.7 K 2024-11-09T22:40:19,870 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for e7b04b07f99af4a950a5462bdc4e6dc3 in 682ms, sequenceid=78, compaction requested=false 2024-11-09T22:40:19,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:19,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:19,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-09T22:40:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-09T22:40:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-09T22:40:19,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 838 msec 2024-11-09T22:40:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 850 msec 2024-11-09T22:40:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,152 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:20,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:40:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T22:40:20,165 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-09T22:40:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T22:40:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,170 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,172 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:20,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110912046de3162b408cbeb2faa9de775adb_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192020150/Put/seqid=0 2024-11-09T22:40:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741971_1147 (size=24358) 2024-11-09T22:40:20,257 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:20,263 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110912046de3162b408cbeb2faa9de775adb_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110912046de3162b408cbeb2faa9de775adb_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:20,264 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/84ff96ac54cd46e99f118f02de433d0d, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:20,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/84ff96ac54cd46e99f118f02de433d0d is 175, key is test_row_0/A:col10/1731192020150/Put/seqid=0 2024-11-09T22:40:20,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741972_1148 (size=73990) 2024-11-09T22:40:20,273 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/84ff96ac54cd46e99f118f02de433d0d 2024-11-09T22:40:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T22:40:20,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/d33899ce40494124bd4e0cdc17fc54e4 is 50, key is test_row_0/B:col10/1731192020150/Put/seqid=0 2024-11-09T22:40:20,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192080276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192080281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192080284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192080291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741973_1149 (size=9657) 2024-11-09T22:40:20,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T22:40:20,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:20,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192080395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192080396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192080405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192080411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T22:40:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:20,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T22:40:20,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192080615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192080615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192080616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192080625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T22:40:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/d33899ce40494124bd4e0cdc17fc54e4 2024-11-09T22:40:20,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/92125cf8bb644f608389514bcd2e4455 is 50, key is test_row_0/C:col10/1731192020150/Put/seqid=0 2024-11-09T22:40:20,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T22:40:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741974_1150 (size=9657) 2024-11-09T22:40:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T22:40:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192080928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T22:40:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:20,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:20,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:20,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192080935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192080935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192080935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:20,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T22:40:21,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:21,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:21,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:21,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:21,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:21,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/92125cf8bb644f608389514bcd2e4455 2024-11-09T22:40:21,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/84ff96ac54cd46e99f118f02de433d0d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/84ff96ac54cd46e99f118f02de433d0d 2024-11-09T22:40:21,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/84ff96ac54cd46e99f118f02de433d0d, entries=400, sequenceid=92, filesize=72.3 K 2024-11-09T22:40:21,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/d33899ce40494124bd4e0cdc17fc54e4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d33899ce40494124bd4e0cdc17fc54e4 2024-11-09T22:40:21,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d33899ce40494124bd4e0cdc17fc54e4, entries=100, sequenceid=92, filesize=9.4 K 2024-11-09T22:40:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/92125cf8bb644f608389514bcd2e4455 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/92125cf8bb644f608389514bcd2e4455 2024-11-09T22:40:21,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/92125cf8bb644f608389514bcd2e4455, entries=100, sequenceid=92, filesize=9.4 K 2024-11-09T22:40:21,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1090ms, sequenceid=92, compaction requested=true 2024-11-09T22:40:21,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:21,243 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:21,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:21,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:21,243 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:21,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:21,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:21,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:21,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:21,246 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:21,246 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:21,246 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/A is initiating minor compaction (all files) 2024-11-09T22:40:21,246 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/B is initiating minor compaction (all files) 2024-11-09T22:40:21,246 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/B in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:21,246 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/A in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:21,246 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/db4731e58cc648438a0420073c9ee3ac, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/60ef9d7a6b8241b4820f771a268e025d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d33899ce40494124bd4e0cdc17fc54e4] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=33.0 K 2024-11-09T22:40:21,246 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/103e9ac949624aa78281488b4a21322f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/8d7b227fe37a47c8b946f3656746bbbd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/84ff96ac54cd46e99f118f02de433d0d] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=132.8 K 2024-11-09T22:40:21,246 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:21,246 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/103e9ac949624aa78281488b4a21322f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/8d7b227fe37a47c8b946f3656746bbbd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/84ff96ac54cd46e99f118f02de433d0d] 2024-11-09T22:40:21,246 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting db4731e58cc648438a0420073c9ee3ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731192017096 2024-11-09T22:40:21,247 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 103e9ac949624aa78281488b4a21322f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731192017096 2024-11-09T22:40:21,247 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ef9d7a6b8241b4820f771a268e025d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731192017860 2024-11-09T22:40:21,247 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d7b227fe37a47c8b946f3656746bbbd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731192017860 2024-11-09T22:40:21,248 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d33899ce40494124bd4e0cdc17fc54e4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192020143 2024-11-09T22:40:21,248 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84ff96ac54cd46e99f118f02de433d0d, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192020122 2024-11-09T22:40:21,258 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:21,258 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/d892797d01f644f2bd2366d5323d15dc is 50, key is test_row_0/B:col10/1731192020150/Put/seqid=0 2024-11-09T22:40:21,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T22:40:21,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:21,262 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T22:40:21,294 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T22:40:21,334 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110975d423267e574448afcf520d4cf37018_e7b04b07f99af4a950a5462bdc4e6dc3 store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:21,337 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110975d423267e574448afcf520d4cf37018_e7b04b07f99af4a950a5462bdc4e6dc3, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:21,338 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110975d423267e574448afcf520d4cf37018_e7b04b07f99af4a950a5462bdc4e6dc3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:21,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e5d23f621d724f49af1446697859f301_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192020277/Put/seqid=0 2024-11-09T22:40:21,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741975_1151 (size=12207) 2024-11-09T22:40:21,367 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/d892797d01f644f2bd2366d5323d15dc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d892797d01f644f2bd2366d5323d15dc 2024-11-09T22:40:21,381 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/B of e7b04b07f99af4a950a5462bdc4e6dc3 into d892797d01f644f2bd2366d5323d15dc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:21,382 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:21,382 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/B, priority=13, startTime=1731192021243; duration=0sec 2024-11-09T22:40:21,382 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:21,382 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:B 2024-11-09T22:40:21,382 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:21,385 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:21,386 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/C is initiating minor compaction (all files) 2024-11-09T22:40:21,386 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/C in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:21,386 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/c2dfa43e4d6047c586d0b4db696681ac, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/15b1590348e54407bc9ba4d9aa2ef7c6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/92125cf8bb644f608389514bcd2e4455] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=33.0 K 2024-11-09T22:40:21,386 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c2dfa43e4d6047c586d0b4db696681ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731192017096 2024-11-09T22:40:21,387 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b1590348e54407bc9ba4d9aa2ef7c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731192017860 2024-11-09T22:40:21,387 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 92125cf8bb644f608389514bcd2e4455, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192020143 2024-11-09T22:40:21,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741976_1152 (size=4469) 2024-11-09T22:40:21,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741977_1153 (size=12154) 2024-11-09T22:40:21,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:21,436 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#C#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:21,437 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/19a86ffed02f4c7c884ffba4b35c6a60 is 50, key is test_row_0/C:col10/1731192020150/Put/seqid=0 2024-11-09T22:40:21,438 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#A#compaction#130 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:21,439 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/7bb4e64e1a5b45c483736bd9edb51556 is 175, key is test_row_0/A:col10/1731192020150/Put/seqid=0 2024-11-09T22:40:21,446 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e5d23f621d724f49af1446697859f301_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e5d23f621d724f49af1446697859f301_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/01e62b40212345d1a7865bf12b44a085, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/01e62b40212345d1a7865bf12b44a085 is 175, key is test_row_0/A:col10/1731192020277/Put/seqid=0 2024-11-09T22:40:21,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:21,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741978_1154 (size=31268) 2024-11-09T22:40:21,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192081474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192081476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192081476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741980_1156 (size=30955) 2024-11-09T22:40:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192081483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,489 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/01e62b40212345d1a7865bf12b44a085 2024-11-09T22:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741979_1155 (size=12207) 2024-11-09T22:40:21,512 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/19a86ffed02f4c7c884ffba4b35c6a60 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/19a86ffed02f4c7c884ffba4b35c6a60 2024-11-09T22:40:21,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/0c38de1ab464449f9be9581c7ab4952c is 50, key is test_row_0/B:col10/1731192020277/Put/seqid=0 2024-11-09T22:40:21,519 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/C of e7b04b07f99af4a950a5462bdc4e6dc3 into 19a86ffed02f4c7c884ffba4b35c6a60(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:21,519 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:21,519 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/C, priority=13, startTime=1731192021245; duration=0sec 2024-11-09T22:40:21,519 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:21,519 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:C 2024-11-09T22:40:21,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741981_1157 (size=12001) 2024-11-09T22:40:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192081595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192081595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192081595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192081595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192081805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192081805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192081805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192081805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:21,877 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/7bb4e64e1a5b45c483736bd9edb51556 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7bb4e64e1a5b45c483736bd9edb51556 2024-11-09T22:40:21,883 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/A of e7b04b07f99af4a950a5462bdc4e6dc3 into 7bb4e64e1a5b45c483736bd9edb51556(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:21,883 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:21,883 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/A, priority=13, startTime=1731192021243; duration=0sec 2024-11-09T22:40:21,883 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:21,883 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:A 2024-11-09T22:40:21,975 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/0c38de1ab464449f9be9581c7ab4952c 2024-11-09T22:40:21,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/96fc556a159043918227078908504f57 is 50, key is test_row_0/C:col10/1731192020277/Put/seqid=0 2024-11-09T22:40:22,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741982_1158 (size=12001) 2024-11-09T22:40:22,004 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/96fc556a159043918227078908504f57 2024-11-09T22:40:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/01e62b40212345d1a7865bf12b44a085 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/01e62b40212345d1a7865bf12b44a085 2024-11-09T22:40:22,021 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/01e62b40212345d1a7865bf12b44a085, entries=150, sequenceid=118, filesize=30.2 K 2024-11-09T22:40:22,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/0c38de1ab464449f9be9581c7ab4952c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0c38de1ab464449f9be9581c7ab4952c 2024-11-09T22:40:22,026 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0c38de1ab464449f9be9581c7ab4952c, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T22:40:22,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/96fc556a159043918227078908504f57 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/96fc556a159043918227078908504f57 2024-11-09T22:40:22,034 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/96fc556a159043918227078908504f57, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T22:40:22,035 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e7b04b07f99af4a950a5462bdc4e6dc3 in 772ms, sequenceid=118, compaction requested=false 2024-11-09T22:40:22,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:22,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-09T22:40:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-09T22:40:22,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-09T22:40:22,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-11-09T22:40:22,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8730 sec 2024-11-09T22:40:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:22,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:40:22,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096da94609e0b743de96d0c2b32b1cbf7d_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192021480/Put/seqid=0 2024-11-09T22:40:22,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741983_1159 (size=14644) 2024-11-09T22:40:22,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192082190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192082190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192082190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192082192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192082305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192082305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192082306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192082306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T22:40:22,315 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:22,315 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-09T22:40:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T22:40:22,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:22,319 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:22,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T22:40:22,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T22:40:22,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:22,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192082516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192082516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192082516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192082516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,540 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:22,545 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096da94609e0b743de96d0c2b32b1cbf7d_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096da94609e0b743de96d0c2b32b1cbf7d_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:22,548 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/ed02c0ea45b842e1bf284e8144bd8158, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:22,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/ed02c0ea45b842e1bf284e8144bd8158 is 175, key is test_row_0/A:col10/1731192021480/Put/seqid=0 2024-11-09T22:40:22,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741984_1160 (size=39599) 2024-11-09T22:40:22,594 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/ed02c0ea45b842e1bf284e8144bd8158 2024-11-09T22:40:22,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/952fdd32a21a47e0b45849e86dcd785f is 50, key is test_row_0/B:col10/1731192021480/Put/seqid=0 2024-11-09T22:40:22,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T22:40:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T22:40:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741985_1161 (size=12051) 2024-11-09T22:40:22,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/952fdd32a21a47e0b45849e86dcd785f 2024-11-09T22:40:22,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/73520c762064472989d2b9e6194bf298 is 50, key is test_row_0/C:col10/1731192021480/Put/seqid=0 2024-11-09T22:40:22,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741986_1162 (size=12051) 2024-11-09T22:40:22,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/73520c762064472989d2b9e6194bf298 2024-11-09T22:40:22,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/ed02c0ea45b842e1bf284e8144bd8158 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed02c0ea45b842e1bf284e8144bd8158 2024-11-09T22:40:22,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed02c0ea45b842e1bf284e8144bd8158, entries=200, sequenceid=132, filesize=38.7 K 2024-11-09T22:40:22,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/952fdd32a21a47e0b45849e86dcd785f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/952fdd32a21a47e0b45849e86dcd785f 2024-11-09T22:40:22,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/952fdd32a21a47e0b45849e86dcd785f, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T22:40:22,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/73520c762064472989d2b9e6194bf298 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/73520c762064472989d2b9e6194bf298 2024-11-09T22:40:22,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/73520c762064472989d2b9e6194bf298, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T22:40:22,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e7b04b07f99af4a950a5462bdc4e6dc3 in 612ms, sequenceid=132, compaction requested=true 2024-11-09T22:40:22,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:22,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:22,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:22,730 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:22,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:22,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:22,731 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:22,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:22,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:22,734 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:22,734 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101822 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:22,734 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/A is initiating minor compaction (all files) 2024-11-09T22:40:22,734 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/B is initiating minor compaction (all files) 2024-11-09T22:40:22,734 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/B in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,734 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/A in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,734 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d892797d01f644f2bd2366d5323d15dc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0c38de1ab464449f9be9581c7ab4952c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/952fdd32a21a47e0b45849e86dcd785f] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=35.4 K 2024-11-09T22:40:22,734 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7bb4e64e1a5b45c483736bd9edb51556, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/01e62b40212345d1a7865bf12b44a085, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed02c0ea45b842e1bf284e8144bd8158] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=99.4 K 2024-11-09T22:40:22,734 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,734 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7bb4e64e1a5b45c483736bd9edb51556, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/01e62b40212345d1a7865bf12b44a085, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed02c0ea45b842e1bf284e8144bd8158] 2024-11-09T22:40:22,735 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d892797d01f644f2bd2366d5323d15dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192017865 2024-11-09T22:40:22,735 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb4e64e1a5b45c483736bd9edb51556, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192017865 2024-11-09T22:40:22,736 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01e62b40212345d1a7865bf12b44a085, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731192020268 2024-11-09T22:40:22,736 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c38de1ab464449f9be9581c7ab4952c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731192020268 2024-11-09T22:40:22,736 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 952fdd32a21a47e0b45849e86dcd785f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731192021464 2024-11-09T22:40:22,736 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed02c0ea45b842e1bf284e8144bd8158, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731192021464 2024-11-09T22:40:22,759 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:22,774 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:22,774 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/71d834a19fc843e698feebaf18fada76 is 50, key is test_row_0/B:col10/1731192021480/Put/seqid=0 2024-11-09T22:40:22,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T22:40:22,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:22,780 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:40:22,783 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110956a7b87a578441708cb55ed8ab0cf4e8_e7b04b07f99af4a950a5462bdc4e6dc3 store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:22,786 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110956a7b87a578441708cb55ed8ab0cf4e8_e7b04b07f99af4a950a5462bdc4e6dc3, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:22,786 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110956a7b87a578441708cb55ed8ab0cf4e8_e7b04b07f99af4a950a5462bdc4e6dc3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:22,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741987_1163 (size=12359) 2024-11-09T22:40:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093e8bb60114074315838a27ac8ee943f0_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192022152/Put/seqid=0 2024-11-09T22:40:22,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:22,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192082837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192082838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741988_1164 (size=4469) 2024-11-09T22:40:22,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192082841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192082842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741989_1165 (size=12304) 2024-11-09T22:40:22,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:22,862 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093e8bb60114074315838a27ac8ee943f0_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e8bb60114074315838a27ac8ee943f0_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:22,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/85b5fbdd392b415db702bd41e735ef9c, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:22,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/85b5fbdd392b415db702bd41e735ef9c is 175, key is test_row_0/A:col10/1731192022152/Put/seqid=0 2024-11-09T22:40:22,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741990_1166 (size=31105) 2024-11-09T22:40:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T22:40:22,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192082945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192082945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192082956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192082956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192083155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192083155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192083155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,159 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in e7b04b07f99af4a950a5462bdc4e6dc3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192083165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192083166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,227 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/71d834a19fc843e698feebaf18fada76 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/71d834a19fc843e698feebaf18fada76 2024-11-09T22:40:23,239 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/B of e7b04b07f99af4a950a5462bdc4e6dc3 into 71d834a19fc843e698feebaf18fada76(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:23,239 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:23,239 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/B, priority=13, startTime=1731192022730; duration=0sec 2024-11-09T22:40:23,239 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:23,239 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:B 2024-11-09T22:40:23,239 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:23,240 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:23,241 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/C is initiating minor compaction (all files) 2024-11-09T22:40:23,241 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/C in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:23,241 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/19a86ffed02f4c7c884ffba4b35c6a60, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/96fc556a159043918227078908504f57, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/73520c762064472989d2b9e6194bf298] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=35.4 K 2024-11-09T22:40:23,241 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a86ffed02f4c7c884ffba4b35c6a60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192017865 2024-11-09T22:40:23,241 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 96fc556a159043918227078908504f57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731192020268 2024-11-09T22:40:23,242 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 73520c762064472989d2b9e6194bf298, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731192021464 2024-11-09T22:40:23,248 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:23,249 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/b7195444873944c6a4e6fa7b0bf71b70 is 50, key is test_row_0/C:col10/1731192021480/Put/seqid=0 2024-11-09T22:40:23,249 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:23,250 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/5b7f6914b04b4c519de0c169ff05d364 is 175, key is test_row_0/A:col10/1731192021480/Put/seqid=0 2024-11-09T22:40:23,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741991_1167 (size=12359) 2024-11-09T22:40:23,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741992_1168 (size=31313) 2024-11-09T22:40:23,275 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/85b5fbdd392b415db702bd41e735ef9c 2024-11-09T22:40:23,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/a64408d905e24bcbaa5a41684d146ab5 is 50, key is test_row_0/B:col10/1731192022152/Put/seqid=0 2024-11-09T22:40:23,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741993_1169 (size=12151) 2024-11-09T22:40:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T22:40:23,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192083465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192083465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192083475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:23,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192083475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:23,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:23,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:23,662 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/b7195444873944c6a4e6fa7b0bf71b70 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/b7195444873944c6a4e6fa7b0bf71b70 2024-11-09T22:40:23,665 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/5b7f6914b04b4c519de0c169ff05d364 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/5b7f6914b04b4c519de0c169ff05d364 2024-11-09T22:40:23,671 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/C of e7b04b07f99af4a950a5462bdc4e6dc3 into b7195444873944c6a4e6fa7b0bf71b70(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:23,671 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:23,671 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/C, priority=13, startTime=1731192022731; duration=0sec 2024-11-09T22:40:23,671 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:23,671 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:C 2024-11-09T22:40:23,675 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/A of e7b04b07f99af4a950a5462bdc4e6dc3 into 5b7f6914b04b4c519de0c169ff05d364(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:23,675 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:23,675 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/A, priority=13, startTime=1731192022730; duration=0sec 2024-11-09T22:40:23,675 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:23,675 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:A 2024-11-09T22:40:23,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/a64408d905e24bcbaa5a41684d146ab5 2024-11-09T22:40:23,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/d346f40e622e4c7a8ec7d27bfba1cc20 is 50, key is test_row_0/C:col10/1731192022152/Put/seqid=0 2024-11-09T22:40:23,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741994_1170 (size=12151) 2024-11-09T22:40:23,732 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/d346f40e622e4c7a8ec7d27bfba1cc20 2024-11-09T22:40:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/85b5fbdd392b415db702bd41e735ef9c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/85b5fbdd392b415db702bd41e735ef9c 2024-11-09T22:40:23,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/85b5fbdd392b415db702bd41e735ef9c, entries=150, sequenceid=157, filesize=30.4 K 2024-11-09T22:40:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/a64408d905e24bcbaa5a41684d146ab5 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a64408d905e24bcbaa5a41684d146ab5 2024-11-09T22:40:23,757 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a64408d905e24bcbaa5a41684d146ab5, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T22:40:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/d346f40e622e4c7a8ec7d27bfba1cc20 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/d346f40e622e4c7a8ec7d27bfba1cc20 2024-11-09T22:40:23,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/d346f40e622e4c7a8ec7d27bfba1cc20, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T22:40:23,768 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e7b04b07f99af4a950a5462bdc4e6dc3 in 988ms, sequenceid=157, compaction requested=false 2024-11-09T22:40:23,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:23,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:23,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-09T22:40:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-09T22:40:23,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-09T22:40:23,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4500 sec 2024-11-09T22:40:23,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4560 sec 2024-11-09T22:40:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:23,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:23,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:40:23,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091041c219c94b4b2194176fe37355e6d3_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192022836/Put/seqid=0 2024-11-09T22:40:24,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741995_1171 (size=14794) 2024-11-09T22:40:24,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192084100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192084134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192084134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192084134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192084245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192084245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192084245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192084245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:24,416 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091041c219c94b4b2194176fe37355e6d3_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091041c219c94b4b2194176fe37355e6d3_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:24,418 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/290fddffbcc541fe862798549450cec4, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:24,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/290fddffbcc541fe862798549450cec4 is 175, key is test_row_0/A:col10/1731192022836/Put/seqid=0 2024-11-09T22:40:24,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741996_1172 (size=39749) 2024-11-09T22:40:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192084455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192084455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192084455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T22:40:24,465 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:24,465 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-09T22:40:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T22:40:24,467 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:24,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192084465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,468 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T22:40:24,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T22:40:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:24,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192084765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192084765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192084765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T22:40:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:24,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:24,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:24,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:24,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192084775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:24,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:24,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T22:40:24,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/290fddffbcc541fe862798549450cec4 2024-11-09T22:40:24,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/daeecea56be54364933eb25fb5cf551f is 50, key is test_row_0/B:col10/1731192022836/Put/seqid=0 2024-11-09T22:40:24,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741997_1173 (size=12151) 2024-11-09T22:40:24,893 INFO [master/9d8c37854f6e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T22:40:24,893 INFO [master/9d8c37854f6e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T22:40:24,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T22:40:24,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:24,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:24,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:24,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:25,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T22:40:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:25,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:25,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T22:40:25,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T22:40:25,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:25,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:25,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:25,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/daeecea56be54364933eb25fb5cf551f 2024-11-09T22:40:25,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/e4d586077cb3428c99ea228c57b96a96 is 50, key is test_row_0/C:col10/1731192022836/Put/seqid=0 2024-11-09T22:40:25,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741998_1174 (size=12151) 2024-11-09T22:40:25,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/e4d586077cb3428c99ea228c57b96a96 2024-11-09T22:40:25,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192085276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:25,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:25,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:25,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:25,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192085285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:25,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192085285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:25,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/290fddffbcc541fe862798549450cec4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/290fddffbcc541fe862798549450cec4 2024-11-09T22:40:25,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/290fddffbcc541fe862798549450cec4, entries=200, sequenceid=172, filesize=38.8 K 2024-11-09T22:40:25,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/daeecea56be54364933eb25fb5cf551f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/daeecea56be54364933eb25fb5cf551f 2024-11-09T22:40:25,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192085295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:25,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/daeecea56be54364933eb25fb5cf551f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T22:40:25,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/e4d586077cb3428c99ea228c57b96a96 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/e4d586077cb3428c99ea228c57b96a96 2024-11-09T22:40:25,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/e4d586077cb3428c99ea228c57b96a96, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T22:40:25,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1340ms, sequenceid=172, compaction requested=true 2024-11-09T22:40:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:25,317 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:25,317 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:25,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:25,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:25,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:25,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:25,319 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:25,319 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/B is initiating minor compaction (all files) 2024-11-09T22:40:25,319 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/B in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,319 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/71d834a19fc843e698feebaf18fada76, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a64408d905e24bcbaa5a41684d146ab5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/daeecea56be54364933eb25fb5cf551f] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=35.8 K 2024-11-09T22:40:25,321 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102167 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:25,321 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/A is initiating minor compaction (all files) 2024-11-09T22:40:25,322 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/A in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,322 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/5b7f6914b04b4c519de0c169ff05d364, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/85b5fbdd392b415db702bd41e735ef9c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/290fddffbcc541fe862798549450cec4] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=99.8 K 2024-11-09T22:40:25,322 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,322 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/5b7f6914b04b4c519de0c169ff05d364, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/85b5fbdd392b415db702bd41e735ef9c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/290fddffbcc541fe862798549450cec4] 2024-11-09T22:40:25,323 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d834a19fc843e698feebaf18fada76, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731192021464 2024-11-09T22:40:25,323 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b7f6914b04b4c519de0c169ff05d364, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731192021464 2024-11-09T22:40:25,324 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting a64408d905e24bcbaa5a41684d146ab5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731192022149 2024-11-09T22:40:25,324 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b5fbdd392b415db702bd41e735ef9c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731192022149 2024-11-09T22:40:25,324 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting daeecea56be54364933eb25fb5cf551f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192022833 2024-11-09T22:40:25,325 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 290fddffbcc541fe862798549450cec4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192022833 2024-11-09T22:40:25,339 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:25,339 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:25,340 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/aa9458ce159047bb99549b56369995e3 is 50, key is test_row_0/B:col10/1731192022836/Put/seqid=0 2024-11-09T22:40:25,347 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109413c4bd862f64368be4a8ee34aa8127b_e7b04b07f99af4a950a5462bdc4e6dc3 store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:25,349 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109413c4bd862f64368be4a8ee34aa8127b_e7b04b07f99af4a950a5462bdc4e6dc3, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:25,350 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109413c4bd862f64368be4a8ee34aa8127b_e7b04b07f99af4a950a5462bdc4e6dc3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:25,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741999_1175 (size=12561) 2024-11-09T22:40:25,378 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/aa9458ce159047bb99549b56369995e3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/aa9458ce159047bb99549b56369995e3 2024-11-09T22:40:25,385 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/B of e7b04b07f99af4a950a5462bdc4e6dc3 into aa9458ce159047bb99549b56369995e3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:25,385 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:25,385 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/B, priority=13, startTime=1731192025317; duration=0sec 2024-11-09T22:40:25,386 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:25,386 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:B 2024-11-09T22:40:25,386 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:25,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742000_1176 (size=4469) 2024-11-09T22:40:25,390 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:25,390 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/C is initiating minor compaction (all files) 2024-11-09T22:40:25,390 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/C in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,391 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/b7195444873944c6a4e6fa7b0bf71b70, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/d346f40e622e4c7a8ec7d27bfba1cc20, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/e4d586077cb3428c99ea228c57b96a96] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=35.8 K 2024-11-09T22:40:25,391 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting b7195444873944c6a4e6fa7b0bf71b70, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731192021464 2024-11-09T22:40:25,392 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d346f40e622e4c7a8ec7d27bfba1cc20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731192022149 2024-11-09T22:40:25,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T22:40:25,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:25,393 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T22:40:25,394 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting e4d586077cb3428c99ea228c57b96a96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192022833 2024-11-09T22:40:25,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091cd6a72464824d3faa8386fbbd4fa0e5_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192024100/Put/seqid=0 2024-11-09T22:40:25,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742001_1177 (size=12304) 2024-11-09T22:40:25,417 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:25,417 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/2fa915f234c744369394c46948322cc8 is 50, key is test_row_0/C:col10/1731192022836/Put/seqid=0 2024-11-09T22:40:25,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742002_1178 (size=12561) 2024-11-09T22:40:25,439 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/2fa915f234c744369394c46948322cc8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2fa915f234c744369394c46948322cc8 2024-11-09T22:40:25,450 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/C of e7b04b07f99af4a950a5462bdc4e6dc3 into 2fa915f234c744369394c46948322cc8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:25,450 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:25,450 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/C, priority=13, startTime=1731192025318; duration=0sec 2024-11-09T22:40:25,450 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:25,450 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:C 2024-11-09T22:40:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T22:40:25,791 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#A#compaction#148 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:25,791 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/81ae498dae7e4da3bfa0c5bbd732af9a is 175, key is test_row_0/A:col10/1731192022836/Put/seqid=0 2024-11-09T22:40:25,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742003_1179 (size=31515) 2024-11-09T22:40:25,811 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/81ae498dae7e4da3bfa0c5bbd732af9a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/81ae498dae7e4da3bfa0c5bbd732af9a 2024-11-09T22:40:25,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:25,818 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/A of e7b04b07f99af4a950a5462bdc4e6dc3 into 81ae498dae7e4da3bfa0c5bbd732af9a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:25,818 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:25,818 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/A, priority=13, startTime=1731192025317; duration=0sec 2024-11-09T22:40:25,818 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:25,818 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:A 2024-11-09T22:40:25,823 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091cd6a72464824d3faa8386fbbd4fa0e5_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091cd6a72464824d3faa8386fbbd4fa0e5_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:25,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/7162a5bbe0f940dc84024c4d17f781dd, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:25,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/7162a5bbe0f940dc84024c4d17f781dd is 175, key is test_row_0/A:col10/1731192024100/Put/seqid=0 2024-11-09T22:40:25,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742004_1180 (size=31105) 2024-11-09T22:40:26,230 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/7162a5bbe0f940dc84024c4d17f781dd 2024-11-09T22:40:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/26fcab4540fc4b3587ebb4d46dd3b1b4 is 50, key is test_row_0/B:col10/1731192024100/Put/seqid=0 2024-11-09T22:40:26,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742005_1181 (size=12151) 2024-11-09T22:40:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:26,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192086316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192086317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192086317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192086319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192086437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192086438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192086438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192086438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T22:40:26,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192086646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192086646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192086646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192086646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:26,660 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/26fcab4540fc4b3587ebb4d46dd3b1b4 2024-11-09T22:40:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/1f01ae8d82c449d68f166bfd181f4f51 is 50, key is test_row_0/C:col10/1731192024100/Put/seqid=0 2024-11-09T22:40:26,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742006_1182 (size=12151) 2024-11-09T22:40:26,701 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/1f01ae8d82c449d68f166bfd181f4f51 2024-11-09T22:40:26,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/7162a5bbe0f940dc84024c4d17f781dd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7162a5bbe0f940dc84024c4d17f781dd 2024-11-09T22:40:26,720 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7162a5bbe0f940dc84024c4d17f781dd, entries=150, sequenceid=197, filesize=30.4 K 2024-11-09T22:40:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/26fcab4540fc4b3587ebb4d46dd3b1b4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/26fcab4540fc4b3587ebb4d46dd3b1b4 2024-11-09T22:40:26,730 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/26fcab4540fc4b3587ebb4d46dd3b1b4, entries=150, sequenceid=197, filesize=11.9 K 2024-11-09T22:40:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/1f01ae8d82c449d68f166bfd181f4f51 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1f01ae8d82c449d68f166bfd181f4f51 2024-11-09T22:40:26,738 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1f01ae8d82c449d68f166bfd181f4f51, entries=150, sequenceid=197, filesize=11.9 K 2024-11-09T22:40:26,739 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1346ms, sequenceid=197, compaction requested=false 2024-11-09T22:40:26,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:26,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:26,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-09T22:40:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-09T22:40:26,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-09T22:40:26,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2750 sec 2024-11-09T22:40:26,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.2840 sec 2024-11-09T22:40:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:26,961 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:26,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T22:40:26,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d64d387230e9451d9b1039374af73222_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192026316/Put/seqid=0 2024-11-09T22:40:26,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742007_1183 (size=12304) 2024-11-09T22:40:26,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:27,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192086995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192086996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192086996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d64d387230e9451d9b1039374af73222_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d64d387230e9451d9b1039374af73222_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:27,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192086997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/b4bc3f2523214cef92dc16222b6172af, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:27,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/b4bc3f2523214cef92dc16222b6172af is 175, key is test_row_0/A:col10/1731192026316/Put/seqid=0 2024-11-09T22:40:27,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742008_1184 (size=31105) 2024-11-09T22:40:27,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192087105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192087115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192087115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192087115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192087315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192087325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192087325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192087325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,434 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/b4bc3f2523214cef92dc16222b6172af 2024-11-09T22:40:27,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/a14d676c1c7c489089bfe0a518833896 is 50, key is test_row_0/B:col10/1731192026316/Put/seqid=0 2024-11-09T22:40:27,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742009_1185 (size=12151) 2024-11-09T22:40:27,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/a14d676c1c7c489089bfe0a518833896 2024-11-09T22:40:27,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/9384810cae134c3e9e8a8538ab51164e is 50, key is test_row_0/C:col10/1731192026316/Put/seqid=0 2024-11-09T22:40:27,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742010_1186 (size=12151) 2024-11-09T22:40:27,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/9384810cae134c3e9e8a8538ab51164e 2024-11-09T22:40:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/b4bc3f2523214cef92dc16222b6172af as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b4bc3f2523214cef92dc16222b6172af 2024-11-09T22:40:27,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b4bc3f2523214cef92dc16222b6172af, entries=150, sequenceid=214, filesize=30.4 K 2024-11-09T22:40:27,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/a14d676c1c7c489089bfe0a518833896 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a14d676c1c7c489089bfe0a518833896 2024-11-09T22:40:27,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a14d676c1c7c489089bfe0a518833896, entries=150, sequenceid=214, filesize=11.9 K 2024-11-09T22:40:27,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/9384810cae134c3e9e8a8538ab51164e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/9384810cae134c3e9e8a8538ab51164e 2024-11-09T22:40:27,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/9384810cae134c3e9e8a8538ab51164e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-09T22:40:27,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e7b04b07f99af4a950a5462bdc4e6dc3 in 560ms, sequenceid=214, compaction requested=true 2024-11-09T22:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:27,522 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:27,522 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:27,523 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:27,523 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/B is initiating minor compaction (all files) 2024-11-09T22:40:27,523 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/B in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:27,523 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/aa9458ce159047bb99549b56369995e3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/26fcab4540fc4b3587ebb4d46dd3b1b4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a14d676c1c7c489089bfe0a518833896] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=36.0 K 2024-11-09T22:40:27,523 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:27,524 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/A is initiating minor compaction (all files) 2024-11-09T22:40:27,524 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/A in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:27,524 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/81ae498dae7e4da3bfa0c5bbd732af9a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7162a5bbe0f940dc84024c4d17f781dd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b4bc3f2523214cef92dc16222b6172af] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=91.5 K 2024-11-09T22:40:27,524 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:27,524 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/81ae498dae7e4da3bfa0c5bbd732af9a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7162a5bbe0f940dc84024c4d17f781dd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b4bc3f2523214cef92dc16222b6172af] 2024-11-09T22:40:27,524 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9458ce159047bb99549b56369995e3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192022833 2024-11-09T22:40:27,524 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ae498dae7e4da3bfa0c5bbd732af9a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192022833 2024-11-09T22:40:27,524 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 26fcab4540fc4b3587ebb4d46dd3b1b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731192024100 2024-11-09T22:40:27,525 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7162a5bbe0f940dc84024c4d17f781dd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731192024100 2024-11-09T22:40:27,525 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting a14d676c1c7c489089bfe0a518833896, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731192026316 2024-11-09T22:40:27,525 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4bc3f2523214cef92dc16222b6172af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731192026316 2024-11-09T22:40:27,541 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:27,541 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/29fc07a4bfe046dc831db58161013688 is 50, key is test_row_0/B:col10/1731192026316/Put/seqid=0 2024-11-09T22:40:27,543 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:27,557 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411090b7a5a732989462089c8f701020c06bc_e7b04b07f99af4a950a5462bdc4e6dc3 store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:27,560 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411090b7a5a732989462089c8f701020c06bc_e7b04b07f99af4a950a5462bdc4e6dc3, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:27,560 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090b7a5a732989462089c8f701020c06bc_e7b04b07f99af4a950a5462bdc4e6dc3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:27,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742011_1187 (size=12663) 2024-11-09T22:40:27,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742012_1188 (size=4469) 2024-11-09T22:40:27,585 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/29fc07a4bfe046dc831db58161013688 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/29fc07a4bfe046dc831db58161013688 2024-11-09T22:40:27,587 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#A#compaction#157 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:27,588 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/d7f03a7e78c84995bbd50338be38f2c1 is 175, key is test_row_0/A:col10/1731192026316/Put/seqid=0 2024-11-09T22:40:27,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742013_1189 (size=31617) 2024-11-09T22:40:27,600 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/B of e7b04b07f99af4a950a5462bdc4e6dc3 into 29fc07a4bfe046dc831db58161013688(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:27,600 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:27,600 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/B, priority=13, startTime=1731192027522; duration=0sec 2024-11-09T22:40:27,600 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:27,600 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:B 2024-11-09T22:40:27,600 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:27,601 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:27,601 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/C is initiating minor compaction (all files) 2024-11-09T22:40:27,601 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/C in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:27,602 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2fa915f234c744369394c46948322cc8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1f01ae8d82c449d68f166bfd181f4f51, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/9384810cae134c3e9e8a8538ab51164e] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=36.0 K 2024-11-09T22:40:27,602 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa915f234c744369394c46948322cc8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192022833 2024-11-09T22:40:27,602 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f01ae8d82c449d68f166bfd181f4f51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731192024100 2024-11-09T22:40:27,602 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9384810cae134c3e9e8a8538ab51164e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731192026316 2024-11-09T22:40:27,624 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:27,625 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/cb6bab12927f40e89e0f83e0e8981860 is 50, key is test_row_0/C:col10/1731192026316/Put/seqid=0 2024-11-09T22:40:27,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:27,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T22:40:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:27,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742014_1190 (size=12663) 2024-11-09T22:40:27,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110965b69edd29064094885685530784320a_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192027627/Put/seqid=0 2024-11-09T22:40:27,656 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/cb6bab12927f40e89e0f83e0e8981860 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/cb6bab12927f40e89e0f83e0e8981860 2024-11-09T22:40:27,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742015_1191 (size=14794) 2024-11-09T22:40:27,701 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/C of e7b04b07f99af4a950a5462bdc4e6dc3 into cb6bab12927f40e89e0f83e0e8981860(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:27,701 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:27,701 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/C, priority=13, startTime=1731192027522; duration=0sec 2024-11-09T22:40:27,701 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:27,701 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:C 2024-11-09T22:40:27,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192087653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192087702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192087702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192087702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192087805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192087815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192087815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192087815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,002 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/d7f03a7e78c84995bbd50338be38f2c1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/d7f03a7e78c84995bbd50338be38f2c1 2024-11-09T22:40:28,007 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/A of e7b04b07f99af4a950a5462bdc4e6dc3 into d7f03a7e78c84995bbd50338be38f2c1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:28,007 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:28,007 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/A, priority=13, startTime=1731192027522; duration=0sec 2024-11-09T22:40:28,007 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:28,007 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:A 2024-11-09T22:40:28,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192088015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192088025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192088025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192088025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:28,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110965b69edd29064094885685530784320a_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110965b69edd29064094885685530784320a_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:28,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/e7c3b54b4f6644f2a1a682db39558c94, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:28,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/e7c3b54b4f6644f2a1a682db39558c94 is 175, key is test_row_0/A:col10/1731192027627/Put/seqid=0 2024-11-09T22:40:28,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742016_1192 (size=39749) 2024-11-09T22:40:28,085 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/e7c3b54b4f6644f2a1a682db39558c94 2024-11-09T22:40:28,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/73905c8853c849bfa31a03f1af1dd0f8 is 50, key is test_row_0/B:col10/1731192027627/Put/seqid=0 2024-11-09T22:40:28,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742017_1193 (size=12151) 2024-11-09T22:40:28,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/73905c8853c849bfa31a03f1af1dd0f8 2024-11-09T22:40:28,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/1d471417bfaa412d9f945ee3e7c29de6 is 50, key is test_row_0/C:col10/1731192027627/Put/seqid=0 2024-11-09T22:40:28,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742018_1194 (size=12151) 2024-11-09T22:40:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192088325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192088335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192088335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192088335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/1d471417bfaa412d9f945ee3e7c29de6 2024-11-09T22:40:28,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/e7c3b54b4f6644f2a1a682db39558c94 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/e7c3b54b4f6644f2a1a682db39558c94 2024-11-09T22:40:28,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/e7c3b54b4f6644f2a1a682db39558c94, entries=200, sequenceid=237, filesize=38.8 K 2024-11-09T22:40:28,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/73905c8853c849bfa31a03f1af1dd0f8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/73905c8853c849bfa31a03f1af1dd0f8 2024-11-09T22:40:28,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/73905c8853c849bfa31a03f1af1dd0f8, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T22:40:28,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/1d471417bfaa412d9f945ee3e7c29de6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1d471417bfaa412d9f945ee3e7c29de6 2024-11-09T22:40:28,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1d471417bfaa412d9f945ee3e7c29de6, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T22:40:28,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for e7b04b07f99af4a950a5462bdc4e6dc3 in 965ms, sequenceid=237, compaction requested=false 2024-11-09T22:40:28,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T22:40:28,625 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:28,625 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-09T22:40:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T22:40:28,639 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:28,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:28,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T22:40:28,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T22:40:28,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:28,794 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T22:40:28,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099ec75a5514c74037b2a9a614ed3c2111_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192027673/Put/seqid=0 2024-11-09T22:40:28,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:28,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742019_1195 (size=12304) 2024-11-09T22:40:28,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192088882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192088889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192088890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192088890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T22:40:28,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192088996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192089005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192089005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192089005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192089205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192089214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192089215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192089215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T22:40:29,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:29,269 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099ec75a5514c74037b2a9a614ed3c2111_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099ec75a5514c74037b2a9a614ed3c2111_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/9edf678471c94c7986ecaeccd35fd7ce, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:29,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/9edf678471c94c7986ecaeccd35fd7ce is 175, key is test_row_0/A:col10/1731192027673/Put/seqid=0 2024-11-09T22:40:29,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742020_1196 (size=31105) 2024-11-09T22:40:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192089515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192089525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192089525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192089525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:29,681 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/9edf678471c94c7986ecaeccd35fd7ce 2024-11-09T22:40:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/3702ec47c6894a558b11ed6bb36c0f63 is 50, key is test_row_0/B:col10/1731192027673/Put/seqid=0 2024-11-09T22:40:29,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742021_1197 (size=12151) 2024-11-09T22:40:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T22:40:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192090024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:30,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192090035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192090035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192090035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:30,102 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/3702ec47c6894a558b11ed6bb36c0f63 2024-11-09T22:40:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/350945a3ab504862b0a2a930f315b48d is 50, key is test_row_0/C:col10/1731192027673/Put/seqid=0 2024-11-09T22:40:30,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742022_1198 (size=12151) 2024-11-09T22:40:30,517 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/350945a3ab504862b0a2a930f315b48d 2024-11-09T22:40:30,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/9edf678471c94c7986ecaeccd35fd7ce as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/9edf678471c94c7986ecaeccd35fd7ce 2024-11-09T22:40:30,526 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/9edf678471c94c7986ecaeccd35fd7ce, entries=150, sequenceid=255, filesize=30.4 K 2024-11-09T22:40:30,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/3702ec47c6894a558b11ed6bb36c0f63 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/3702ec47c6894a558b11ed6bb36c0f63 2024-11-09T22:40:30,530 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/3702ec47c6894a558b11ed6bb36c0f63, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T22:40:30,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/350945a3ab504862b0a2a930f315b48d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/350945a3ab504862b0a2a930f315b48d 2024-11-09T22:40:30,535 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/350945a3ab504862b0a2a930f315b48d, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T22:40:30,536 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1743ms, sequenceid=255, compaction requested=true 2024-11-09T22:40:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-09T22:40:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-09T22:40:30,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-09T22:40:30,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-11-09T22:40:30,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.9150 sec 2024-11-09T22:40:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T22:40:30,786 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:30,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-09T22:40:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T22:40:30,788 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:30,788 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:30,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T22:40:30,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-09T22:40:30,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:30,941 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T22:40:30,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e4c2048e71e24ab58ea9edb33a40fc44_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192028887/Put/seqid=0 2024-11-09T22:40:30,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742023_1199 (size=12454) 2024-11-09T22:40:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:31,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:31,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192091056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192091087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192091087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192091087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T22:40:31,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192091195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192091195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192091195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192091195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:31,356 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e4c2048e71e24ab58ea9edb33a40fc44_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e4c2048e71e24ab58ea9edb33a40fc44_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/be6cc9e5fd484bdc8f5db59aeea02abd, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:31,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/be6cc9e5fd484bdc8f5db59aeea02abd is 175, key is test_row_0/A:col10/1731192028887/Put/seqid=0 2024-11-09T22:40:31,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742024_1200 (size=31255) 2024-11-09T22:40:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192091405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192091405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192091405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192091405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T22:40:31,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192091715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192091715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192091715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192091715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:31,762 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/be6cc9e5fd484bdc8f5db59aeea02abd 2024-11-09T22:40:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/2a146cac3e4447969a060a293c72e7e6 is 50, key is test_row_0/B:col10/1731192028887/Put/seqid=0 2024-11-09T22:40:31,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742025_1201 (size=12301) 2024-11-09T22:40:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T22:40:32,174 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/2a146cac3e4447969a060a293c72e7e6 2024-11-09T22:40:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/df07ff0a5a7642ce9222b86e55db8b16 is 50, key is test_row_0/C:col10/1731192028887/Put/seqid=0 2024-11-09T22:40:32,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742026_1202 (size=12301) 2024-11-09T22:40:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192092225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:32,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192092225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:32,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192092225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192092225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:32,585 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/df07ff0a5a7642ce9222b86e55db8b16 2024-11-09T22:40:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/be6cc9e5fd484bdc8f5db59aeea02abd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/be6cc9e5fd484bdc8f5db59aeea02abd 2024-11-09T22:40:32,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/be6cc9e5fd484bdc8f5db59aeea02abd, entries=150, sequenceid=276, filesize=30.5 K 2024-11-09T22:40:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/2a146cac3e4447969a060a293c72e7e6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/2a146cac3e4447969a060a293c72e7e6 2024-11-09T22:40:32,599 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/2a146cac3e4447969a060a293c72e7e6, entries=150, sequenceid=276, filesize=12.0 K 2024-11-09T22:40:32,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/df07ff0a5a7642ce9222b86e55db8b16 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/df07ff0a5a7642ce9222b86e55db8b16 2024-11-09T22:40:32,604 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/df07ff0a5a7642ce9222b86e55db8b16, entries=150, sequenceid=276, filesize=12.0 K 2024-11-09T22:40:32,605 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1664ms, sequenceid=276, compaction requested=true 2024-11-09T22:40:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-09T22:40:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-09T22:40:32,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-09T22:40:32,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-11-09T22:40:32,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8240 sec 2024-11-09T22:40:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T22:40:32,935 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:32,935 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-09T22:40:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T22:40:32,936 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:32,937 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:32,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T22:40:33,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-09T22:40:33,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:33,089 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T22:40:33,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110930eefb8c57404bd5abd57f7990474211_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192031061/Put/seqid=0 2024-11-09T22:40:33,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742027_1203 (size=12454) 2024-11-09T22:40:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:33,168 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110930eefb8c57404bd5abd57f7990474211_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110930eefb8c57404bd5abd57f7990474211_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:33,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/4a7ad4ff778b40a0a303c0d71179e5b6, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:33,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/4a7ad4ff778b40a0a303c0d71179e5b6 is 175, key is test_row_0/A:col10/1731192031061/Put/seqid=0 2024-11-09T22:40:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:33,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:33,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742028_1204 (size=31255) 2024-11-09T22:40:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T22:40:33,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192093247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192093248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192093249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192093250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192093257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192093366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192093366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192093366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192093366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192093374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T22:40:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192093575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192093575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192093575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192093575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192093585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,621 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/4a7ad4ff778b40a0a303c0d71179e5b6 2024-11-09T22:40:33,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/480af17926a04232b824a685b75b8835 is 50, key is test_row_0/B:col10/1731192031061/Put/seqid=0 2024-11-09T22:40:33,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742029_1205 (size=12301) 2024-11-09T22:40:33,687 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/480af17926a04232b824a685b75b8835 2024-11-09T22:40:33,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/60786e527ac1406db7d234f70e00f948 is 50, key is test_row_0/C:col10/1731192031061/Put/seqid=0 2024-11-09T22:40:33,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742030_1206 (size=12301) 2024-11-09T22:40:33,722 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/60786e527ac1406db7d234f70e00f948 2024-11-09T22:40:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/4a7ad4ff778b40a0a303c0d71179e5b6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4a7ad4ff778b40a0a303c0d71179e5b6 2024-11-09T22:40:33,747 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4a7ad4ff778b40a0a303c0d71179e5b6, entries=150, sequenceid=292, filesize=30.5 K 2024-11-09T22:40:33,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/480af17926a04232b824a685b75b8835 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/480af17926a04232b824a685b75b8835 2024-11-09T22:40:33,762 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/480af17926a04232b824a685b75b8835, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T22:40:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/60786e527ac1406db7d234f70e00f948 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/60786e527ac1406db7d234f70e00f948 2024-11-09T22:40:33,775 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/60786e527ac1406db7d234f70e00f948, entries=150, sequenceid=292, filesize=12.0 K 2024-11-09T22:40:33,779 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e7b04b07f99af4a950a5462bdc4e6dc3 in 690ms, sequenceid=292, compaction requested=true 2024-11-09T22:40:33,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:33,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:33,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-09T22:40:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-09T22:40:33,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-09T22:40:33,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 845 msec 2024-11-09T22:40:33,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 850 msec 2024-11-09T22:40:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:33,888 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:33,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T22:40:33,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192093900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192093902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192093903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192093903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192093905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:33,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091bd894485d194a04b3103babc4280915_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192033887/Put/seqid=0 2024-11-09T22:40:33,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742031_1207 (size=17534) 2024-11-09T22:40:34,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192094015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192094025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192094025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192094025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192094025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T22:40:34,075 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:34,076 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-09T22:40:34,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T22:40:34,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:34,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T22:40:34,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192094225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:34,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:34,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192094235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192094236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192094236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192094236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,345 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:34,352 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091bd894485d194a04b3103babc4280915_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091bd894485d194a04b3103babc4280915_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:34,354 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/bb00b91fb49e4ed9a237da18be8cb8a4, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:34,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/bb00b91fb49e4ed9a237da18be8cb8a4 is 175, key is test_row_0/A:col10/1731192033887/Put/seqid=0 2024-11-09T22:40:34,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:34,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:34,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742032_1208 (size=48639) 2024-11-09T22:40:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T22:40:34,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:34,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:34,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192094534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192094544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192094545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192094545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192094545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:34,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:34,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:34,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T22:40:34,791 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/bb00b91fb49e4ed9a237da18be8cb8a4 2024-11-09T22:40:34,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/fb448b9ec03f44d4993a3c05f74b8321 is 50, key is test_row_0/B:col10/1731192033887/Put/seqid=0 2024-11-09T22:40:34,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742033_1209 (size=12301) 2024-11-09T22:40:34,830 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-09T22:40:34,830 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:34,830 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:34,830 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:34,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:34,834 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-09T22:40:34,834 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:34,834 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:34,834 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:34,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:34,838 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-09T22:40:34,839 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:34,839 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:34,839 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:34,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:34,842 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-09T22:40:34,842 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:34,842 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:34,842 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:34,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:34,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:34,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:34,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:34,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:35,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:35,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36256 deadline: 1731192095055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36250 deadline: 1731192095055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36264 deadline: 1731192095055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:35,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:35,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731192095055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731192095055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:40:35,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:35,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:35,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/fb448b9ec03f44d4993a3c05f74b8321 2024-11-09T22:40:35,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/32bb3fd021b149d8b6a2a91a7c7765cb is 50, key is test_row_0/C:col10/1731192033887/Put/seqid=0 2024-11-09T22:40:35,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742034_1210 (size=12301) 2024-11-09T22:40:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T22:40:35,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. as already flushing 2024-11-09T22:40:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:35,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/32bb3fd021b149d8b6a2a91a7c7765cb 2024-11-09T22:40:35,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/bb00b91fb49e4ed9a237da18be8cb8a4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bb00b91fb49e4ed9a237da18be8cb8a4 2024-11-09T22:40:35,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bb00b91fb49e4ed9a237da18be8cb8a4, entries=250, sequenceid=314, filesize=47.5 K 2024-11-09T22:40:35,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/fb448b9ec03f44d4993a3c05f74b8321 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/fb448b9ec03f44d4993a3c05f74b8321 2024-11-09T22:40:35,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/fb448b9ec03f44d4993a3c05f74b8321, entries=150, sequenceid=314, filesize=12.0 K 2024-11-09T22:40:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/32bb3fd021b149d8b6a2a91a7c7765cb as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/32bb3fd021b149d8b6a2a91a7c7765cb 2024-11-09T22:40:35,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/32bb3fd021b149d8b6a2a91a7c7765cb, entries=150, sequenceid=314, filesize=12.0 K 2024-11-09T22:40:35,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1754ms, sequenceid=314, compaction requested=true 2024-11-09T22:40:35,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:35,642 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T22:40:35,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:35,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:35,643 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T22:40:35,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:35,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:35,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7b04b07f99af4a950a5462bdc4e6dc3:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:35,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:35,645 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73868 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T22:40:35,645 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/B is initiating minor compaction (all files) 2024-11-09T22:40:35,645 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/B in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,645 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/29fc07a4bfe046dc831db58161013688, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/73905c8853c849bfa31a03f1af1dd0f8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/3702ec47c6894a558b11ed6bb36c0f63, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/2a146cac3e4447969a060a293c72e7e6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/480af17926a04232b824a685b75b8835, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/fb448b9ec03f44d4993a3c05f74b8321] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=72.1 K 2024-11-09T22:40:35,646 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 29fc07a4bfe046dc831db58161013688, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731192026316 2024-11-09T22:40:35,646 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 213620 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T22:40:35,646 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/A is initiating minor compaction (all files) 2024-11-09T22:40:35,646 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/A in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,646 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/d7f03a7e78c84995bbd50338be38f2c1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/e7c3b54b4f6644f2a1a682db39558c94, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/9edf678471c94c7986ecaeccd35fd7ce, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/be6cc9e5fd484bdc8f5db59aeea02abd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4a7ad4ff778b40a0a303c0d71179e5b6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bb00b91fb49e4ed9a237da18be8cb8a4] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=208.6 K 2024-11-09T22:40:35,646 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 73905c8853c849bfa31a03f1af1dd0f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731192026992 2024-11-09T22:40:35,646 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,646 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/d7f03a7e78c84995bbd50338be38f2c1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/e7c3b54b4f6644f2a1a682db39558c94, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/9edf678471c94c7986ecaeccd35fd7ce, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/be6cc9e5fd484bdc8f5db59aeea02abd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4a7ad4ff778b40a0a303c0d71179e5b6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bb00b91fb49e4ed9a237da18be8cb8a4] 2024-11-09T22:40:35,647 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3702ec47c6894a558b11ed6bb36c0f63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731192027633 2024-11-09T22:40:35,647 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7f03a7e78c84995bbd50338be38f2c1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731192026316 2024-11-09T22:40:35,647 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a146cac3e4447969a060a293c72e7e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731192028885 2024-11-09T22:40:35,647 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7c3b54b4f6644f2a1a682db39558c94, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731192026992 2024-11-09T22:40:35,647 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 480af17926a04232b824a685b75b8835, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731192031048 2024-11-09T22:40:35,647 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9edf678471c94c7986ecaeccd35fd7ce, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731192027633 2024-11-09T22:40:35,648 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting fb448b9ec03f44d4993a3c05f74b8321, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731192033242 2024-11-09T22:40:35,648 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting be6cc9e5fd484bdc8f5db59aeea02abd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731192028885 2024-11-09T22:40:35,649 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a7ad4ff778b40a0a303c0d71179e5b6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731192031048 2024-11-09T22:40:35,649 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb00b91fb49e4ed9a237da18be8cb8a4, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731192033242 2024-11-09T22:40:35,675 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:35,677 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#B#compaction#174 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:35,678 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/7e0b3fba11c14f198459d949b517baaf is 50, key is test_row_0/B:col10/1731192033887/Put/seqid=0 2024-11-09T22:40:35,687 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110900db33cf63c0464c8979a9130d0a32e4_e7b04b07f99af4a950a5462bdc4e6dc3 store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:35,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742035_1211 (size=13017) 2024-11-09T22:40:35,761 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110900db33cf63c0464c8979a9130d0a32e4_e7b04b07f99af4a950a5462bdc4e6dc3, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:35,761 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110900db33cf63c0464c8979a9130d0a32e4_e7b04b07f99af4a950a5462bdc4e6dc3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:35,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-09T22:40:35,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,768 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T22:40:35,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742036_1212 (size=4469) 2024-11-09T22:40:35,805 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#A#compaction#175 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:35,806 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/bfdfb9acfd8f4d2993a5690e0b6af339 is 175, key is test_row_0/A:col10/1731192033887/Put/seqid=0 2024-11-09T22:40:35,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093c4684d7803745ca92f10161e93b9c2d_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192033898/Put/seqid=0 2024-11-09T22:40:35,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742037_1213 (size=31971) 2024-11-09T22:40:35,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742038_1214 (size=12454) 2024-11-09T22:40:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:35,849 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/bfdfb9acfd8f4d2993a5690e0b6af339 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bfdfb9acfd8f4d2993a5690e0b6af339 2024-11-09T22:40:35,851 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093c4684d7803745ca92f10161e93b9c2d_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c4684d7803745ca92f10161e93b9c2d_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/4ef46314dc0349ec88d9c7f82a172252, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/4ef46314dc0349ec88d9c7f82a172252 is 175, key is test_row_0/A:col10/1731192033898/Put/seqid=0 2024-11-09T22:40:35,860 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/A of e7b04b07f99af4a950a5462bdc4e6dc3 into bfdfb9acfd8f4d2993a5690e0b6af339(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:35,860 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:35,860 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/A, priority=10, startTime=1731192035642; duration=0sec 2024-11-09T22:40:35,860 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:35,860 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:A 2024-11-09T22:40:35,861 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-09T22:40:35,866 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73868 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-09T22:40:35,866 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): e7b04b07f99af4a950a5462bdc4e6dc3/C is initiating minor compaction (all files) 2024-11-09T22:40:35,866 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7b04b07f99af4a950a5462bdc4e6dc3/C in TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:35,866 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/cb6bab12927f40e89e0f83e0e8981860, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1d471417bfaa412d9f945ee3e7c29de6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/350945a3ab504862b0a2a930f315b48d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/df07ff0a5a7642ce9222b86e55db8b16, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/60786e527ac1406db7d234f70e00f948, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/32bb3fd021b149d8b6a2a91a7c7765cb] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp, totalSize=72.1 K 2024-11-09T22:40:35,867 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb6bab12927f40e89e0f83e0e8981860, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731192026316 2024-11-09T22:40:35,867 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d471417bfaa412d9f945ee3e7c29de6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731192026992 2024-11-09T22:40:35,867 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 350945a3ab504862b0a2a930f315b48d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731192027633 2024-11-09T22:40:35,868 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting df07ff0a5a7642ce9222b86e55db8b16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731192028885 2024-11-09T22:40:35,868 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60786e527ac1406db7d234f70e00f948, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731192031048 2024-11-09T22:40:35,869 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32bb3fd021b149d8b6a2a91a7c7765cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731192033242 2024-11-09T22:40:35,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742039_1215 (size=31255) 2024-11-09T22:40:35,892 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/4ef46314dc0349ec88d9c7f82a172252 2024-11-09T22:40:35,909 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7b04b07f99af4a950a5462bdc4e6dc3#C#compaction#177 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:35,910 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/f176b841aa49453b86b327d3f758d264 is 50, key is test_row_0/C:col10/1731192033887/Put/seqid=0 2024-11-09T22:40:35,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/0a13c499825246b8ad8b05e363ff05a7 is 50, key is test_row_0/B:col10/1731192033898/Put/seqid=0 2024-11-09T22:40:35,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742040_1216 (size=13017) 2024-11-09T22:40:35,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742041_1217 (size=12301) 2024-11-09T22:40:35,956 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/0a13c499825246b8ad8b05e363ff05a7 2024-11-09T22:40:35,958 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/f176b841aa49453b86b327d3f758d264 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/f176b841aa49453b86b327d3f758d264 2024-11-09T22:40:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/2eeb45d287d74193aad91d58a481bc16 is 50, key is test_row_0/C:col10/1731192033898/Put/seqid=0 2024-11-09T22:40:35,972 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/C of e7b04b07f99af4a950a5462bdc4e6dc3 into f176b841aa49453b86b327d3f758d264(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:35,972 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:35,972 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/C, priority=10, startTime=1731192035643; duration=0sec 2024-11-09T22:40:35,972 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:35,972 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:C 2024-11-09T22:40:35,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742042_1218 (size=12301) 2024-11-09T22:40:35,983 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/2eeb45d287d74193aad91d58a481bc16 2024-11-09T22:40:35,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/4ef46314dc0349ec88d9c7f82a172252 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4ef46314dc0349ec88d9c7f82a172252 2024-11-09T22:40:36,002 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4ef46314dc0349ec88d9c7f82a172252, entries=150, sequenceid=328, filesize=30.5 K 2024-11-09T22:40:36,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/0a13c499825246b8ad8b05e363ff05a7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0a13c499825246b8ad8b05e363ff05a7 2024-11-09T22:40:36,011 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0a13c499825246b8ad8b05e363ff05a7, entries=150, sequenceid=328, filesize=12.0 K 2024-11-09T22:40:36,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/2eeb45d287d74193aad91d58a481bc16 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2eeb45d287d74193aad91d58a481bc16 2024-11-09T22:40:36,018 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2eeb45d287d74193aad91d58a481bc16, entries=150, sequenceid=328, filesize=12.0 K 2024-11-09T22:40:36,020 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for e7b04b07f99af4a950a5462bdc4e6dc3 in 253ms, sequenceid=328, compaction requested=false 2024-11-09T22:40:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-09T22:40:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-09T22:40:36,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-09T22:40:36,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-11-09T22:40:36,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.9500 sec 2024-11-09T22:40:36,068 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-09T22:40:36,068 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-09T22:40:36,068 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:36,068 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:36,068 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,068 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,068 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,068 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:36,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:36,068 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-09T22:40:36,068 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:36,068 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,068 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:36,070 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-09T22:40:36,070 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:36,070 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,071 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:36,077 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-09T22:40:36,077 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:40:36,077 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,077 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:36,121 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/7e0b3fba11c14f198459d949b517baaf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/7e0b3fba11c14f198459d949b517baaf 2024-11-09T22:40:36,128 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e7b04b07f99af4a950a5462bdc4e6dc3/B of e7b04b07f99af4a950a5462bdc4e6dc3 into 7e0b3fba11c14f198459d949b517baaf(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:36,128 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7b04b07f99af4a950a5462bdc4e6dc3: 2024-11-09T22:40:36,128 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., storeName=e7b04b07f99af4a950a5462bdc4e6dc3/B, priority=10, startTime=1731192035643; duration=0sec 2024-11-09T22:40:36,128 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:36,128 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7b04b07f99af4a950a5462bdc4e6dc3:B 2024-11-09T22:40:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T22:40:36,226 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3188 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3039 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1368 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4099 rows 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1374 2024-11-09T22:40:36,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4114 rows 2024-11-09T22:40:36,226 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:40:36,226 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:40:36,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:36,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:40:36,228 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T22:40:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T22:40:36,232 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192036231"}]},"ts":"1731192036231"} 2024-11-09T22:40:36,234 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T22:40:36,234 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T22:40:36,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:40:36,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, UNASSIGN}] 2024-11-09T22:40:36,238 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, UNASSIGN 2024-11-09T22:40:36,239 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=e7b04b07f99af4a950a5462bdc4e6dc3, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:36,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, UNASSIGN because future has completed 2024-11-09T22:40:36,240 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:40:36,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:40:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T22:40:36,392 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:36,393 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:40:36,393 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing e7b04b07f99af4a950a5462bdc4e6dc3, disabling compactions & flushes 2024-11-09T22:40:36,393 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:36,393 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:36,393 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. after waiting 0 ms 2024-11-09T22:40:36,393 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:36,393 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing e7b04b07f99af4a950a5462bdc4e6dc3 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T22:40:36,400 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099348546b174345b4a1796d71da935a35_e7b04b07f99af4a950a5462bdc4e6dc3 is 50, key is test_row_0/A:col10/1731192036066/Put/seqid=0 2024-11-09T22:40:36,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742043_1219 (size=9914) 2024-11-09T22:40:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T22:40:36,808 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:36,819 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099348546b174345b4a1796d71da935a35_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099348546b174345b4a1796d71da935a35_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:36,821 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/36413faeacd94270bccccdeb201efb5a, store: [table=TestAcidGuarantees family=A region=e7b04b07f99af4a950a5462bdc4e6dc3] 2024-11-09T22:40:36,821 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/36413faeacd94270bccccdeb201efb5a is 175, key is test_row_0/A:col10/1731192036066/Put/seqid=0 2024-11-09T22:40:36,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742044_1220 (size=22561) 2024-11-09T22:40:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T22:40:37,227 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/36413faeacd94270bccccdeb201efb5a 2024-11-09T22:40:37,241 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/b1189c943b494c20833c26acf9add0f9 is 50, key is test_row_0/B:col10/1731192036066/Put/seqid=0 2024-11-09T22:40:37,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742045_1221 (size=9857) 2024-11-09T22:40:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T22:40:37,645 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/b1189c943b494c20833c26acf9add0f9 2024-11-09T22:40:37,655 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/5190271952d746f2ab32a738cfaf86e7 is 50, key is test_row_0/C:col10/1731192036066/Put/seqid=0 2024-11-09T22:40:37,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742046_1222 (size=9857) 2024-11-09T22:40:38,062 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/5190271952d746f2ab32a738cfaf86e7 2024-11-09T22:40:38,072 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/A/36413faeacd94270bccccdeb201efb5a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/36413faeacd94270bccccdeb201efb5a 2024-11-09T22:40:38,079 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/36413faeacd94270bccccdeb201efb5a, entries=100, sequenceid=339, filesize=22.0 K 2024-11-09T22:40:38,080 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/B/b1189c943b494c20833c26acf9add0f9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/b1189c943b494c20833c26acf9add0f9 2024-11-09T22:40:38,084 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/b1189c943b494c20833c26acf9add0f9, entries=100, sequenceid=339, filesize=9.6 K 2024-11-09T22:40:38,085 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/.tmp/C/5190271952d746f2ab32a738cfaf86e7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/5190271952d746f2ab32a738cfaf86e7 2024-11-09T22:40:38,089 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/5190271952d746f2ab32a738cfaf86e7, entries=100, sequenceid=339, filesize=9.6 K 2024-11-09T22:40:38,090 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1697ms, sequenceid=339, compaction requested=true 2024-11-09T22:40:38,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b972422bfd0b447596122ecd07b60389, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed1d91fb432144fa93a6c53c0c08f647, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/67996d8361b545adb3f06d76d1601666, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/103e9ac949624aa78281488b4a21322f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/8d7b227fe37a47c8b946f3656746bbbd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/84ff96ac54cd46e99f118f02de433d0d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7bb4e64e1a5b45c483736bd9edb51556, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/01e62b40212345d1a7865bf12b44a085, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed02c0ea45b842e1bf284e8144bd8158, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/5b7f6914b04b4c519de0c169ff05d364, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/85b5fbdd392b415db702bd41e735ef9c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/290fddffbcc541fe862798549450cec4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/81ae498dae7e4da3bfa0c5bbd732af9a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7162a5bbe0f940dc84024c4d17f781dd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/d7f03a7e78c84995bbd50338be38f2c1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b4bc3f2523214cef92dc16222b6172af, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/e7c3b54b4f6644f2a1a682db39558c94, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/9edf678471c94c7986ecaeccd35fd7ce, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/be6cc9e5fd484bdc8f5db59aeea02abd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4a7ad4ff778b40a0a303c0d71179e5b6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bb00b91fb49e4ed9a237da18be8cb8a4] to archive 2024-11-09T22:40:38,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:40:38,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b972422bfd0b447596122ecd07b60389 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b972422bfd0b447596122ecd07b60389 2024-11-09T22:40:38,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed1d91fb432144fa93a6c53c0c08f647 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed1d91fb432144fa93a6c53c0c08f647 2024-11-09T22:40:38,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/67996d8361b545adb3f06d76d1601666 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/67996d8361b545adb3f06d76d1601666 2024-11-09T22:40:38,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/103e9ac949624aa78281488b4a21322f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/103e9ac949624aa78281488b4a21322f 2024-11-09T22:40:38,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/8d7b227fe37a47c8b946f3656746bbbd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/8d7b227fe37a47c8b946f3656746bbbd 2024-11-09T22:40:38,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/84ff96ac54cd46e99f118f02de433d0d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/84ff96ac54cd46e99f118f02de433d0d 2024-11-09T22:40:38,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7bb4e64e1a5b45c483736bd9edb51556 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7bb4e64e1a5b45c483736bd9edb51556 2024-11-09T22:40:38,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/01e62b40212345d1a7865bf12b44a085 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/01e62b40212345d1a7865bf12b44a085 2024-11-09T22:40:38,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed02c0ea45b842e1bf284e8144bd8158 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/ed02c0ea45b842e1bf284e8144bd8158 2024-11-09T22:40:38,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/5b7f6914b04b4c519de0c169ff05d364 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/5b7f6914b04b4c519de0c169ff05d364 2024-11-09T22:40:38,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/85b5fbdd392b415db702bd41e735ef9c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/85b5fbdd392b415db702bd41e735ef9c 2024-11-09T22:40:38,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/290fddffbcc541fe862798549450cec4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/290fddffbcc541fe862798549450cec4 2024-11-09T22:40:38,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/81ae498dae7e4da3bfa0c5bbd732af9a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/81ae498dae7e4da3bfa0c5bbd732af9a 2024-11-09T22:40:38,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7162a5bbe0f940dc84024c4d17f781dd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/7162a5bbe0f940dc84024c4d17f781dd 2024-11-09T22:40:38,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/d7f03a7e78c84995bbd50338be38f2c1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/d7f03a7e78c84995bbd50338be38f2c1 2024-11-09T22:40:38,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b4bc3f2523214cef92dc16222b6172af to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/b4bc3f2523214cef92dc16222b6172af 2024-11-09T22:40:38,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/e7c3b54b4f6644f2a1a682db39558c94 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/e7c3b54b4f6644f2a1a682db39558c94 2024-11-09T22:40:38,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/9edf678471c94c7986ecaeccd35fd7ce to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/9edf678471c94c7986ecaeccd35fd7ce 2024-11-09T22:40:38,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/be6cc9e5fd484bdc8f5db59aeea02abd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/be6cc9e5fd484bdc8f5db59aeea02abd 2024-11-09T22:40:38,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4a7ad4ff778b40a0a303c0d71179e5b6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4a7ad4ff778b40a0a303c0d71179e5b6 2024-11-09T22:40:38,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bb00b91fb49e4ed9a237da18be8cb8a4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bb00b91fb49e4ed9a237da18be8cb8a4 2024-11-09T22:40:38,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/8383e82c1ee345ca92254592e0f06764, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/9549263903694d74903a52c7efee0d82, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/db4731e58cc648438a0420073c9ee3ac, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/665fcc7e05444ab1a3415c7b07e3e774, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/60ef9d7a6b8241b4820f771a268e025d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d892797d01f644f2bd2366d5323d15dc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d33899ce40494124bd4e0cdc17fc54e4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0c38de1ab464449f9be9581c7ab4952c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/71d834a19fc843e698feebaf18fada76, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/952fdd32a21a47e0b45849e86dcd785f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a64408d905e24bcbaa5a41684d146ab5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/aa9458ce159047bb99549b56369995e3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/daeecea56be54364933eb25fb5cf551f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/26fcab4540fc4b3587ebb4d46dd3b1b4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/29fc07a4bfe046dc831db58161013688, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a14d676c1c7c489089bfe0a518833896, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/73905c8853c849bfa31a03f1af1dd0f8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/3702ec47c6894a558b11ed6bb36c0f63, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/2a146cac3e4447969a060a293c72e7e6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/480af17926a04232b824a685b75b8835, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/fb448b9ec03f44d4993a3c05f74b8321] to archive 2024-11-09T22:40:38,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:40:38,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/8383e82c1ee345ca92254592e0f06764 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/8383e82c1ee345ca92254592e0f06764 2024-11-09T22:40:38,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/9549263903694d74903a52c7efee0d82 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/9549263903694d74903a52c7efee0d82 2024-11-09T22:40:38,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/db4731e58cc648438a0420073c9ee3ac to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/db4731e58cc648438a0420073c9ee3ac 2024-11-09T22:40:38,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/665fcc7e05444ab1a3415c7b07e3e774 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/665fcc7e05444ab1a3415c7b07e3e774 2024-11-09T22:40:38,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/60ef9d7a6b8241b4820f771a268e025d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/60ef9d7a6b8241b4820f771a268e025d 2024-11-09T22:40:38,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d892797d01f644f2bd2366d5323d15dc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d892797d01f644f2bd2366d5323d15dc 2024-11-09T22:40:38,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d33899ce40494124bd4e0cdc17fc54e4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/d33899ce40494124bd4e0cdc17fc54e4 2024-11-09T22:40:38,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0c38de1ab464449f9be9581c7ab4952c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0c38de1ab464449f9be9581c7ab4952c 2024-11-09T22:40:38,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/71d834a19fc843e698feebaf18fada76 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/71d834a19fc843e698feebaf18fada76 2024-11-09T22:40:38,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/952fdd32a21a47e0b45849e86dcd785f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/952fdd32a21a47e0b45849e86dcd785f 2024-11-09T22:40:38,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a64408d905e24bcbaa5a41684d146ab5 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a64408d905e24bcbaa5a41684d146ab5 2024-11-09T22:40:38,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/aa9458ce159047bb99549b56369995e3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/aa9458ce159047bb99549b56369995e3 2024-11-09T22:40:38,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/daeecea56be54364933eb25fb5cf551f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/daeecea56be54364933eb25fb5cf551f 2024-11-09T22:40:38,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/26fcab4540fc4b3587ebb4d46dd3b1b4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/26fcab4540fc4b3587ebb4d46dd3b1b4 2024-11-09T22:40:38,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/29fc07a4bfe046dc831db58161013688 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/29fc07a4bfe046dc831db58161013688 2024-11-09T22:40:38,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a14d676c1c7c489089bfe0a518833896 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/a14d676c1c7c489089bfe0a518833896 2024-11-09T22:40:38,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/73905c8853c849bfa31a03f1af1dd0f8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/73905c8853c849bfa31a03f1af1dd0f8 2024-11-09T22:40:38,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/3702ec47c6894a558b11ed6bb36c0f63 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/3702ec47c6894a558b11ed6bb36c0f63 2024-11-09T22:40:38,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/2a146cac3e4447969a060a293c72e7e6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/2a146cac3e4447969a060a293c72e7e6 2024-11-09T22:40:38,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/480af17926a04232b824a685b75b8835 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/480af17926a04232b824a685b75b8835 2024-11-09T22:40:38,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/fb448b9ec03f44d4993a3c05f74b8321 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/fb448b9ec03f44d4993a3c05f74b8321 2024-11-09T22:40:38,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/59de11c459f84020ace4c22fb0e244f9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/ce70e3acb5c745f2a325724bedbb9574, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/c2dfa43e4d6047c586d0b4db696681ac, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/70bdafa821f14467ae7a58b69d9b3066, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/15b1590348e54407bc9ba4d9aa2ef7c6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/19a86ffed02f4c7c884ffba4b35c6a60, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/92125cf8bb644f608389514bcd2e4455, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/96fc556a159043918227078908504f57, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/b7195444873944c6a4e6fa7b0bf71b70, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/73520c762064472989d2b9e6194bf298, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/d346f40e622e4c7a8ec7d27bfba1cc20, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2fa915f234c744369394c46948322cc8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/e4d586077cb3428c99ea228c57b96a96, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1f01ae8d82c449d68f166bfd181f4f51, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/cb6bab12927f40e89e0f83e0e8981860, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/9384810cae134c3e9e8a8538ab51164e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1d471417bfaa412d9f945ee3e7c29de6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/350945a3ab504862b0a2a930f315b48d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/df07ff0a5a7642ce9222b86e55db8b16, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/60786e527ac1406db7d234f70e00f948, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/32bb3fd021b149d8b6a2a91a7c7765cb] to archive 2024-11-09T22:40:38,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:40:38,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/59de11c459f84020ace4c22fb0e244f9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/59de11c459f84020ace4c22fb0e244f9 2024-11-09T22:40:38,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/ce70e3acb5c745f2a325724bedbb9574 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/ce70e3acb5c745f2a325724bedbb9574 2024-11-09T22:40:38,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/c2dfa43e4d6047c586d0b4db696681ac to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/c2dfa43e4d6047c586d0b4db696681ac 2024-11-09T22:40:38,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/70bdafa821f14467ae7a58b69d9b3066 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/70bdafa821f14467ae7a58b69d9b3066 2024-11-09T22:40:38,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/15b1590348e54407bc9ba4d9aa2ef7c6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/15b1590348e54407bc9ba4d9aa2ef7c6 2024-11-09T22:40:38,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/19a86ffed02f4c7c884ffba4b35c6a60 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/19a86ffed02f4c7c884ffba4b35c6a60 2024-11-09T22:40:38,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/92125cf8bb644f608389514bcd2e4455 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/92125cf8bb644f608389514bcd2e4455 2024-11-09T22:40:38,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/96fc556a159043918227078908504f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/96fc556a159043918227078908504f57 2024-11-09T22:40:38,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/b7195444873944c6a4e6fa7b0bf71b70 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/b7195444873944c6a4e6fa7b0bf71b70 2024-11-09T22:40:38,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/73520c762064472989d2b9e6194bf298 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/73520c762064472989d2b9e6194bf298 2024-11-09T22:40:38,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/d346f40e622e4c7a8ec7d27bfba1cc20 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/d346f40e622e4c7a8ec7d27bfba1cc20 2024-11-09T22:40:38,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2fa915f234c744369394c46948322cc8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2fa915f234c744369394c46948322cc8 2024-11-09T22:40:38,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/e4d586077cb3428c99ea228c57b96a96 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/e4d586077cb3428c99ea228c57b96a96 2024-11-09T22:40:38,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1f01ae8d82c449d68f166bfd181f4f51 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1f01ae8d82c449d68f166bfd181f4f51 2024-11-09T22:40:38,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/cb6bab12927f40e89e0f83e0e8981860 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/cb6bab12927f40e89e0f83e0e8981860 2024-11-09T22:40:38,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/9384810cae134c3e9e8a8538ab51164e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/9384810cae134c3e9e8a8538ab51164e 2024-11-09T22:40:38,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1d471417bfaa412d9f945ee3e7c29de6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/1d471417bfaa412d9f945ee3e7c29de6 2024-11-09T22:40:38,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/350945a3ab504862b0a2a930f315b48d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/350945a3ab504862b0a2a930f315b48d 2024-11-09T22:40:38,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/df07ff0a5a7642ce9222b86e55db8b16 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/df07ff0a5a7642ce9222b86e55db8b16 2024-11-09T22:40:38,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/60786e527ac1406db7d234f70e00f948 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/60786e527ac1406db7d234f70e00f948 2024-11-09T22:40:38,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/32bb3fd021b149d8b6a2a91a7c7765cb to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/32bb3fd021b149d8b6a2a91a7c7765cb 2024-11-09T22:40:38,159 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=4 2024-11-09T22:40:38,160 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. 2024-11-09T22:40:38,160 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for e7b04b07f99af4a950a5462bdc4e6dc3: Waiting for close lock at 1731192036393Running coprocessor pre-close hooks at 1731192036393Disabling compacts and flushes for region at 1731192036393Disabling writes for close at 1731192036393Obtaining lock to block concurrent updates at 1731192036393Preparing flush snapshotting stores in e7b04b07f99af4a950a5462bdc4e6dc3 at 1731192036393Finished memstore snapshotting TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731192036393Flushing stores of TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3. at 1731192036394 (+1 ms)Flushing e7b04b07f99af4a950a5462bdc4e6dc3/A: creating writer at 1731192036394Flushing mob file e7b04b07f99af4a950a5462bdc4e6dc3/A: appending metadata at 1731192036400 (+6 ms)Flushing mob file e7b04b07f99af4a950a5462bdc4e6dc3/A: closing flushed file at 1731192036400Flushing e7b04b07f99af4a950a5462bdc4e6dc3/A: appending metadata at 1731192036821 (+421 ms)Flushing e7b04b07f99af4a950a5462bdc4e6dc3/A: closing flushed file at 1731192036821Flushing e7b04b07f99af4a950a5462bdc4e6dc3/B: creating writer at 1731192037236 (+415 ms)Flushing e7b04b07f99af4a950a5462bdc4e6dc3/B: appending metadata at 1731192037240 (+4 ms)Flushing e7b04b07f99af4a950a5462bdc4e6dc3/B: closing flushed file at 1731192037240Flushing e7b04b07f99af4a950a5462bdc4e6dc3/C: creating writer at 1731192037651 (+411 ms)Flushing e7b04b07f99af4a950a5462bdc4e6dc3/C: appending metadata at 1731192037655 (+4 ms)Flushing e7b04b07f99af4a950a5462bdc4e6dc3/C: closing flushed file at 1731192037655Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1898c2f5: reopening flushed file at 1731192038071 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a0c17a4: reopening flushed file at 1731192038079 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42245ca9: reopening flushed file at 1731192038084 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e7b04b07f99af4a950a5462bdc4e6dc3 in 1697ms, sequenceid=339, compaction requested=true at 1731192038090 (+6 ms)Writing region close event to WAL at 1731192038156 (+66 ms)Running coprocessor post-close hooks at 1731192038159 (+3 ms)Closed at 1731192038160 (+1 ms) 2024-11-09T22:40:38,161 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,162 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=e7b04b07f99af4a950a5462bdc4e6dc3, regionState=CLOSED 2024-11-09T22:40:38,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:40:38,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-09T22:40:38,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure e7b04b07f99af4a950a5462bdc4e6dc3, server=9d8c37854f6e,35101,1731191981027 in 1.9230 sec 2024-11-09T22:40:38,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-09T22:40:38,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7b04b07f99af4a950a5462bdc4e6dc3, UNASSIGN in 1.9270 sec 2024-11-09T22:40:38,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T22:40:38,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9310 sec 2024-11-09T22:40:38,169 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192038168"}]},"ts":"1731192038168"} 2024-11-09T22:40:38,170 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T22:40:38,170 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T22:40:38,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9420 sec 2024-11-09T22:40:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-09T22:40:38,385 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:38,386 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T22:40:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:38,388 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T22:40:38,390 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:38,395 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,398 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/recovered.edits] 2024-11-09T22:40:38,400 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/36413faeacd94270bccccdeb201efb5a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/36413faeacd94270bccccdeb201efb5a 2024-11-09T22:40:38,401 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4ef46314dc0349ec88d9c7f82a172252 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/4ef46314dc0349ec88d9c7f82a172252 2024-11-09T22:40:38,402 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bfdfb9acfd8f4d2993a5690e0b6af339 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/A/bfdfb9acfd8f4d2993a5690e0b6af339 2024-11-09T22:40:38,405 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0a13c499825246b8ad8b05e363ff05a7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/0a13c499825246b8ad8b05e363ff05a7 2024-11-09T22:40:38,405 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/7e0b3fba11c14f198459d949b517baaf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/7e0b3fba11c14f198459d949b517baaf 2024-11-09T22:40:38,407 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/b1189c943b494c20833c26acf9add0f9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/B/b1189c943b494c20833c26acf9add0f9 2024-11-09T22:40:38,409 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2eeb45d287d74193aad91d58a481bc16 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/2eeb45d287d74193aad91d58a481bc16 2024-11-09T22:40:38,410 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/5190271952d746f2ab32a738cfaf86e7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/5190271952d746f2ab32a738cfaf86e7 2024-11-09T22:40:38,411 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/f176b841aa49453b86b327d3f758d264 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/C/f176b841aa49453b86b327d3f758d264 2024-11-09T22:40:38,413 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/recovered.edits/342.seqid to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3/recovered.edits/342.seqid 2024-11-09T22:40:38,414 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,414 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T22:40:38,415 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T22:40:38,415 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T22:40:38,419 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091041c219c94b4b2194176fe37355e6d3_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091041c219c94b4b2194176fe37355e6d3_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,420 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110912046de3162b408cbeb2faa9de775adb_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110912046de3162b408cbeb2faa9de775adb_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,421 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091bd894485d194a04b3103babc4280915_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091bd894485d194a04b3103babc4280915_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,422 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091cd6a72464824d3faa8386fbbd4fa0e5_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091cd6a72464824d3faa8386fbbd4fa0e5_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,423 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110930eefb8c57404bd5abd57f7990474211_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110930eefb8c57404bd5abd57f7990474211_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,424 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c4684d7803745ca92f10161e93b9c2d_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093c4684d7803745ca92f10161e93b9c2d_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,425 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e8bb60114074315838a27ac8ee943f0_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e8bb60114074315838a27ac8ee943f0_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,426 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110965b69edd29064094885685530784320a_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110965b69edd29064094885685530784320a_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,427 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096da94609e0b743de96d0c2b32b1cbf7d_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096da94609e0b743de96d0c2b32b1cbf7d_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,428 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099348546b174345b4a1796d71da935a35_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099348546b174345b4a1796d71da935a35_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,429 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099ec75a5514c74037b2a9a614ed3c2111_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099ec75a5514c74037b2a9a614ed3c2111_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,430 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a1976dc1a3c64d8bbee9f74ae3af7fda_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a1976dc1a3c64d8bbee9f74ae3af7fda_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,431 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ced95a9e7c144cf498c9d671af29ce50_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ced95a9e7c144cf498c9d671af29ce50_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,431 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d64d387230e9451d9b1039374af73222_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d64d387230e9451d9b1039374af73222_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,432 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e4c2048e71e24ab58ea9edb33a40fc44_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e4c2048e71e24ab58ea9edb33a40fc44_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,433 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e5d23f621d724f49af1446697859f301_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e5d23f621d724f49af1446697859f301_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,434 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ece38e63031c475e8125d324493766ba_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ece38e63031c475e8125d324493766ba_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,435 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fe795347e5e24a9eb0af9f0581bc2700_e7b04b07f99af4a950a5462bdc4e6dc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fe795347e5e24a9eb0af9f0581bc2700_e7b04b07f99af4a950a5462bdc4e6dc3 2024-11-09T22:40:38,435 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T22:40:38,438 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:38,439 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T22:40:38,441 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T22:40:38,442 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:38,442 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T22:40:38,442 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731192038442"}]},"ts":"9223372036854775807"} 2024-11-09T22:40:38,444 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T22:40:38,444 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e7b04b07f99af4a950a5462bdc4e6dc3, NAME => 'TestAcidGuarantees,,1731192011558.e7b04b07f99af4a950a5462bdc4e6dc3.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T22:40:38,444 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T22:40:38,445 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731192038444"}]},"ts":"9223372036854775807"} 2024-11-09T22:40:38,446 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T22:40:38,447 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:38,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-09T22:40:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-09T22:40:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T22:40:38,495 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:38,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=244 (was 239) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1065910943_22 at /127.0.0.1:33564 [Waiting for operation #303] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1065910943_22 at /127.0.0.1:55344 [Waiting for operation #639] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-870303366_22 at /127.0.0.1:49580 [Waiting for operation #550] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=597 (was 499) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17928 (was 17992) 2024-11-09T22:40:38,510 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=244, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=597, ProcessCount=11, AvailableMemoryMB=17928 2024-11-09T22:40:38,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:40:38,512 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:40:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T22:40:38,513 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T22:40:38,513 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:38,513 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-09T22:40:38,514 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T22:40:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-09T22:40:38,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742047_1223 (size=959) 2024-11-09T22:40:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-09T22:40:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-09T22:40:38,923 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 2024-11-09T22:40:38,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742048_1224 (size=53) 2024-11-09T22:40:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-09T22:40:39,244 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T22:40:39,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:40:39,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3a78d83b1d743c9a9aaf5a1e71b1bc87, disabling compactions & flushes 2024-11-09T22:40:39,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:39,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:39,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. after waiting 0 ms 2024-11-09T22:40:39,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:39,334 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:39,334 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: Waiting for close lock at 1731192039333Disabling compacts and flushes for region at 1731192039333Disabling writes for close at 1731192039333Writing region close event to WAL at 1731192039333Closed at 1731192039333 2024-11-09T22:40:39,336 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T22:40:39,337 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731192039336"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731192039336"}]},"ts":"1731192039336"} 2024-11-09T22:40:39,341 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T22:40:39,342 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T22:40:39,342 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192039342"}]},"ts":"1731192039342"} 2024-11-09T22:40:39,345 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T22:40:39,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, ASSIGN}] 2024-11-09T22:40:39,347 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, ASSIGN 2024-11-09T22:40:39,348 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, ASSIGN; state=OFFLINE, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=false 2024-11-09T22:40:39,499 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=3a78d83b1d743c9a9aaf5a1e71b1bc87, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:39,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, ASSIGN because future has completed 2024-11-09T22:40:39,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:40:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-09T22:40:39,660 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:39,660 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:40:39,660 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,660 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:40:39,661 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,661 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,664 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,666 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a78d83b1d743c9a9aaf5a1e71b1bc87 columnFamilyName A 2024-11-09T22:40:39,666 DEBUG [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:39,667 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] regionserver.HStore(327): Store=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:39,667 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,669 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a78d83b1d743c9a9aaf5a1e71b1bc87 columnFamilyName B 2024-11-09T22:40:39,669 DEBUG [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:39,670 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] regionserver.HStore(327): Store=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:39,670 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,671 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a78d83b1d743c9a9aaf5a1e71b1bc87 columnFamilyName C 2024-11-09T22:40:39,671 DEBUG [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:40:39,672 INFO [StoreOpener-3a78d83b1d743c9a9aaf5a1e71b1bc87-1 {}] regionserver.HStore(327): Store=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:40:39,672 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,673 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,673 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,674 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,674 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,675 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:40:39,676 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,678 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:40:39,679 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 3a78d83b1d743c9a9aaf5a1e71b1bc87; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69511199, jitterRate=0.03579758107662201}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:40:39,679 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:39,680 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: Running coprocessor pre-open hook at 1731192039661Writing region info on filesystem at 1731192039661Initializing all the Stores at 1731192039663 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192039663Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192039663Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192039663Cleaning up temporary data from old regions at 1731192039674 (+11 ms)Running coprocessor post-open hooks at 1731192039679 (+5 ms)Region opened successfully at 1731192039680 (+1 ms) 2024-11-09T22:40:39,681 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., pid=67, masterSystemTime=1731192039654 2024-11-09T22:40:39,683 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:39,683 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:39,683 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=3a78d83b1d743c9a9aaf5a1e71b1bc87, regionState=OPEN, openSeqNum=2, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:39,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:40:39,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-09T22:40:39,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 in 184 msec 2024-11-09T22:40:39,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-09T22:40:39,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, ASSIGN in 343 msec 2024-11-09T22:40:39,691 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T22:40:39,692 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192039691"}]},"ts":"1731192039691"} 2024-11-09T22:40:39,693 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T22:40:39,695 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T22:40:39,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-09T22:40:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-09T22:40:40,665 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:40,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7dbd0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60011061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,672 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T22:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d8c37854f6e,34651,1731191980160 2024-11-09T22:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@603905f 2024-11-09T22:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T22:40:40,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T22:40:40,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26322d0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690fc41d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,682 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d4d256, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,684 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e6ba5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,685 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39343491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,687 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271ef2ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2309e893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,689 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5577e8bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,691 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0632fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,692 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a92c0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,694 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@501ed283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,695 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1431df47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,697 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54afb83c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,698 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39466daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df924b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fa97c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63344e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603a650c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:40:40,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:40:40,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:40:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:40:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:40:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b70083a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:40:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:40:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:40:40,706 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:40:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f813f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:40:40,707 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,707 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,707 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,707 DEBUG [Thread-1063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,707 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,708 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,708 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:40,708 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,708 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,709 DEBUG [Thread-1063 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,709 DEBUG [Thread-1071 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,709 DEBUG [Thread-1069 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,709 DEBUG [Thread-1063 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,709 DEBUG [Thread-1069 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-09T22:40:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,711 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T22:40:40,711 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:40:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,711 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:40,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,713 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:40,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:40:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:40:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:40:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:40:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:40,724 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:40,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:40:40,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/8eea8512a6f2475a803cb42d2ac321f2 is 50, key is test_row_0/A:col10/1731192040724/Put/seqid=0 2024-11-09T22:40:40,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192100746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192100747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192100749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192100754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192100751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742049_1225 (size=12001) 2024-11-09T22:40:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T22:40:40,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T22:40:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192100855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:40,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:40,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:40,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:40,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192100865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192100865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192100865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192100865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T22:40:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T22:40:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192101076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192101076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192101076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192101076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192101085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/8eea8512a6f2475a803cb42d2ac321f2 2024-11-09T22:40:41,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T22:40:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/21a6d6706970461eb4886f3ae33fd7f6 is 50, key is test_row_0/B:col10/1731192040724/Put/seqid=0 2024-11-09T22:40:41,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742050_1226 (size=12001) 2024-11-09T22:40:41,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T22:40:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T22:40:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192101384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192101384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192101384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192101385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192101395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T22:40:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/21a6d6706970461eb4886f3ae33fd7f6 2024-11-09T22:40:41,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ec4d7f35a0814e09ba3ab069a4dab030 is 50, key is test_row_0/C:col10/1731192040724/Put/seqid=0 2024-11-09T22:40:41,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742051_1227 (size=12001) 2024-11-09T22:40:41,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T22:40:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ec4d7f35a0814e09ba3ab069a4dab030 2024-11-09T22:40:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:41,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/8eea8512a6f2475a803cb42d2ac321f2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/8eea8512a6f2475a803cb42d2ac321f2 2024-11-09T22:40:41,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/8eea8512a6f2475a803cb42d2ac321f2, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T22:40:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/21a6d6706970461eb4886f3ae33fd7f6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/21a6d6706970461eb4886f3ae33fd7f6 2024-11-09T22:40:41,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/21a6d6706970461eb4886f3ae33fd7f6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T22:40:41,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ec4d7f35a0814e09ba3ab069a4dab030 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ec4d7f35a0814e09ba3ab069a4dab030 2024-11-09T22:40:41,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ec4d7f35a0814e09ba3ab069a4dab030, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T22:40:41,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 945ms, sequenceid=14, compaction requested=false 2024-11-09T22:40:41,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:41,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T22:40:41,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:41,791 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T22:40:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/f969111511bc455eaafecf1dad41c11f is 50, key is test_row_0/A:col10/1731192040751/Put/seqid=0 2024-11-09T22:40:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T22:40:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742052_1228 (size=12001) 2024-11-09T22:40:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:41,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:41,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192101905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192101905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192101906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192101909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:41,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192101911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192102015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192102015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192102016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192102025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192102225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192102225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192102225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192102235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,274 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/f969111511bc455eaafecf1dad41c11f 2024-11-09T22:40:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/fe6ca0b27f584a8988aed30d8f095898 is 50, key is test_row_0/B:col10/1731192040751/Put/seqid=0 2024-11-09T22:40:42,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742053_1229 (size=12001) 2024-11-09T22:40:42,303 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/fe6ca0b27f584a8988aed30d8f095898 2024-11-09T22:40:42,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/9ea7ff72c87b4c80843392391e4d6fd7 is 50, key is test_row_0/C:col10/1731192040751/Put/seqid=0 2024-11-09T22:40:42,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742054_1230 (size=12001) 2024-11-09T22:40:42,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/9ea7ff72c87b4c80843392391e4d6fd7 2024-11-09T22:40:42,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/f969111511bc455eaafecf1dad41c11f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/f969111511bc455eaafecf1dad41c11f 2024-11-09T22:40:42,364 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/f969111511bc455eaafecf1dad41c11f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T22:40:42,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/fe6ca0b27f584a8988aed30d8f095898 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/fe6ca0b27f584a8988aed30d8f095898 2024-11-09T22:40:42,375 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/fe6ca0b27f584a8988aed30d8f095898, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T22:40:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/9ea7ff72c87b4c80843392391e4d6fd7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9ea7ff72c87b4c80843392391e4d6fd7 2024-11-09T22:40:42,399 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9ea7ff72c87b4c80843392391e4d6fd7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T22:40:42,400 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 609ms, sequenceid=37, compaction requested=false 2024-11-09T22:40:42,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:42,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:42,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-09T22:40:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-09T22:40:42,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-09T22:40:42,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6920 sec 2024-11-09T22:40:42,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.7040 sec 2024-11-09T22:40:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:42,540 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:42,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T22:40:42,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c6a12eb6e42e4d7f85170258caa1d355 is 50, key is test_row_0/A:col10/1731192042538/Put/seqid=0 2024-11-09T22:40:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192102575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192102582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192102583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192102584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742055_1231 (size=14341) 2024-11-09T22:40:42,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192102695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192102695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192102695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192102695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T22:40:42,865 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:42,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-09T22:40:42,869 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T22:40:42,870 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:42,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:42,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192102906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192102906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192102906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192102906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192102925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:42,929 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 3a78d83b1d743c9a9aaf5a1e71b1bc87 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:40:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T22:40:42,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c6a12eb6e42e4d7f85170258caa1d355 2024-11-09T22:40:43,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/acb5fc40ea784271bd06ca35a48a5330 is 50, key is test_row_0/B:col10/1731192042538/Put/seqid=0 2024-11-09T22:40:43,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T22:40:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742056_1232 (size=12001) 2024-11-09T22:40:43,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T22:40:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T22:40:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192103215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192103215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192103225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192103225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T22:40:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/acb5fc40ea784271bd06ca35a48a5330 2024-11-09T22:40:43,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T22:40:43,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/34d17135e47742a99a3be89cc0ac6484 is 50, key is test_row_0/C:col10/1731192042538/Put/seqid=0 2024-11-09T22:40:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T22:40:43,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742057_1233 (size=12001) 2024-11-09T22:40:43,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/34d17135e47742a99a3be89cc0ac6484 2024-11-09T22:40:43,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c6a12eb6e42e4d7f85170258caa1d355 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c6a12eb6e42e4d7f85170258caa1d355 2024-11-09T22:40:43,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c6a12eb6e42e4d7f85170258caa1d355, entries=200, sequenceid=52, filesize=14.0 K 2024-11-09T22:40:43,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/acb5fc40ea784271bd06ca35a48a5330 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/acb5fc40ea784271bd06ca35a48a5330 2024-11-09T22:40:43,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/acb5fc40ea784271bd06ca35a48a5330, entries=150, sequenceid=52, filesize=11.7 K 2024-11-09T22:40:43,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/34d17135e47742a99a3be89cc0ac6484 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/34d17135e47742a99a3be89cc0ac6484 2024-11-09T22:40:43,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/34d17135e47742a99a3be89cc0ac6484, entries=150, sequenceid=52, filesize=11.7 K 2024-11-09T22:40:43,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1016ms, sequenceid=52, compaction requested=true 2024-11-09T22:40:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:43,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:43,557 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:43,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:43,557 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:43,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:43,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:43,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:43,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:43,559 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:43,559 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:43,559 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:40:43,559 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:40:43,559 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,559 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/21a6d6706970461eb4886f3ae33fd7f6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/fe6ca0b27f584a8988aed30d8f095898, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/acb5fc40ea784271bd06ca35a48a5330] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=35.2 K 2024-11-09T22:40:43,559 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,559 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/8eea8512a6f2475a803cb42d2ac321f2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/f969111511bc455eaafecf1dad41c11f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c6a12eb6e42e4d7f85170258caa1d355] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=37.4 K 2024-11-09T22:40:43,560 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eea8512a6f2475a803cb42d2ac321f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731192040721 2024-11-09T22:40:43,560 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a6d6706970461eb4886f3ae33fd7f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731192040721 2024-11-09T22:40:43,560 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f969111511bc455eaafecf1dad41c11f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731192040737 2024-11-09T22:40:43,561 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting fe6ca0b27f584a8988aed30d8f095898, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731192040737 2024-11-09T22:40:43,561 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a12eb6e42e4d7f85170258caa1d355, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731192041898 2024-11-09T22:40:43,561 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting acb5fc40ea784271bd06ca35a48a5330, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731192041898 2024-11-09T22:40:43,593 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:43,594 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/af8c91dde83f490ea7805844475ddeff is 50, key is test_row_0/B:col10/1731192042538/Put/seqid=0 2024-11-09T22:40:43,606 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#193 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:43,608 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/84eef6ee590e47c2a74e250473c94a10 is 50, key is test_row_0/A:col10/1731192042538/Put/seqid=0 2024-11-09T22:40:43,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742058_1234 (size=12104) 2024-11-09T22:40:43,638 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/af8c91dde83f490ea7805844475ddeff as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/af8c91dde83f490ea7805844475ddeff 2024-11-09T22:40:43,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T22:40:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,639 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T22:40:43,644 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into af8c91dde83f490ea7805844475ddeff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:43,644 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:43,644 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192043557; duration=0sec 2024-11-09T22:40:43,645 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:43,645 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:40:43,645 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:43,646 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:43,646 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/C is initiating minor compaction (all files) 2024-11-09T22:40:43,646 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/C in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:43,646 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ec4d7f35a0814e09ba3ab069a4dab030, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9ea7ff72c87b4c80843392391e4d6fd7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/34d17135e47742a99a3be89cc0ac6484] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=35.2 K 2024-11-09T22:40:43,647 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4d7f35a0814e09ba3ab069a4dab030, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731192040721 2024-11-09T22:40:43,647 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea7ff72c87b4c80843392391e4d6fd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731192040737 2024-11-09T22:40:43,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742059_1235 (size=12104) 2024-11-09T22:40:43,648 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d17135e47742a99a3be89cc0ac6484, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731192041898 2024-11-09T22:40:43,654 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/84eef6ee590e47c2a74e250473c94a10 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/84eef6ee590e47c2a74e250473c94a10 2024-11-09T22:40:43,662 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 84eef6ee590e47c2a74e250473c94a10(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:43,663 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:43,663 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192043556; duration=0sec 2024-11-09T22:40:43,663 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:43,663 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:40:43,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c58dae23bf014ed8a95987e91a86d83d is 50, key is test_row_0/A:col10/1731192042582/Put/seqid=0 2024-11-09T22:40:43,677 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#C#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:43,677 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ebecd7d6f5e444caa149eedac6568857 is 50, key is test_row_0/C:col10/1731192042538/Put/seqid=0 2024-11-09T22:40:43,696 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T22:40:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:43,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742060_1236 (size=12001) 2024-11-09T22:40:43,741 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c58dae23bf014ed8a95987e91a86d83d 2024-11-09T22:40:43,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192103763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192103766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192103763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192103768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742061_1237 (size=12104) 2024-11-09T22:40:43,783 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ebecd7d6f5e444caa149eedac6568857 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ebecd7d6f5e444caa149eedac6568857 2024-11-09T22:40:43,790 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/C of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into ebecd7d6f5e444caa149eedac6568857(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:43,791 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:43,791 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, priority=13, startTime=1731192043558; duration=0sec 2024-11-09T22:40:43,791 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:43,791 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:40:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/6e2db26d703249c4aa5c30749f2c8a98 is 50, key is test_row_0/B:col10/1731192042582/Put/seqid=0 2024-11-09T22:40:43,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742062_1238 (size=12001) 2024-11-09T22:40:43,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192103875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192103876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192103885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192103885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T22:40:44,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192104086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192104086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192104095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192104095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,217 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/6e2db26d703249c4aa5c30749f2c8a98 2024-11-09T22:40:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/9080dad29eae4d18a30a0e32370fc308 is 50, key is test_row_0/C:col10/1731192042582/Put/seqid=0 2024-11-09T22:40:44,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742063_1239 (size=12001) 2024-11-09T22:40:44,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192104394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192104394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192104405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192104405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/9080dad29eae4d18a30a0e32370fc308 2024-11-09T22:40:44,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c58dae23bf014ed8a95987e91a86d83d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c58dae23bf014ed8a95987e91a86d83d 2024-11-09T22:40:44,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c58dae23bf014ed8a95987e91a86d83d, entries=150, sequenceid=73, filesize=11.7 K 2024-11-09T22:40:44,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/6e2db26d703249c4aa5c30749f2c8a98 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6e2db26d703249c4aa5c30749f2c8a98 2024-11-09T22:40:44,679 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6e2db26d703249c4aa5c30749f2c8a98, entries=150, sequenceid=73, filesize=11.7 K 2024-11-09T22:40:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/9080dad29eae4d18a30a0e32370fc308 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9080dad29eae4d18a30a0e32370fc308 2024-11-09T22:40:44,688 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9080dad29eae4d18a30a0e32370fc308, entries=150, sequenceid=73, filesize=11.7 K 2024-11-09T22:40:44,691 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1052ms, sequenceid=73, compaction requested=false 2024-11-09T22:40:44,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:44,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:44,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-09T22:40:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-09T22:40:44,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-09T22:40:44,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8230 sec 2024-11-09T22:40:44,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8300 sec 2024-11-09T22:40:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:44,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T22:40:44,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/e5a6512a6dc045278ce352f25bdc3255 is 50, key is test_row_0/A:col10/1731192044906/Put/seqid=0 2024-11-09T22:40:44,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742064_1240 (size=14341) 2024-11-09T22:40:44,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/e5a6512a6dc045278ce352f25bdc3255 2024-11-09T22:40:44,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/ff48429dfb594dc0a8a7df1e39030084 is 50, key is test_row_0/B:col10/1731192044906/Put/seqid=0 2024-11-09T22:40:44,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742065_1241 (size=12001) 2024-11-09T22:40:44,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/ff48429dfb594dc0a8a7df1e39030084 2024-11-09T22:40:44,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/e65cefb2da2247fc8bd56d4690418e27 is 50, key is test_row_0/C:col10/1731192044906/Put/seqid=0 2024-11-09T22:40:44,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192104935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192104936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742066_1242 (size=12001) 2024-11-09T22:40:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192104939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192104941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192104945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:44,951 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 3a78d83b1d743c9a9aaf5a1e71b1bc87 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:40:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T22:40:45,021 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:45,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-09T22:40:45,023 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:45,024 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:45,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T22:40:45,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192105045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192105045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192105045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192105055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T22:40:45,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T22:40:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:45,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192105255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192105256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192105256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192105265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T22:40:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:45,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:45,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/e65cefb2da2247fc8bd56d4690418e27 2024-11-09T22:40:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T22:40:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/e5a6512a6dc045278ce352f25bdc3255 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e5a6512a6dc045278ce352f25bdc3255 2024-11-09T22:40:45,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e5a6512a6dc045278ce352f25bdc3255, entries=200, sequenceid=93, filesize=14.0 K 2024-11-09T22:40:45,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/ff48429dfb594dc0a8a7df1e39030084 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ff48429dfb594dc0a8a7df1e39030084 2024-11-09T22:40:45,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ff48429dfb594dc0a8a7df1e39030084, entries=150, sequenceid=93, filesize=11.7 K 2024-11-09T22:40:45,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/e65cefb2da2247fc8bd56d4690418e27 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e65cefb2da2247fc8bd56d4690418e27 2024-11-09T22:40:45,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e65cefb2da2247fc8bd56d4690418e27, entries=150, sequenceid=93, filesize=11.7 K 2024-11-09T22:40:45,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 508ms, sequenceid=93, compaction requested=true 2024-11-09T22:40:45,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:45,415 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:45,416 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:45,417 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:45,417 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:40:45,417 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:45,417 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/84eef6ee590e47c2a74e250473c94a10, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c58dae23bf014ed8a95987e91a86d83d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e5a6512a6dc045278ce352f25bdc3255] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=37.5 K 2024-11-09T22:40:45,418 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:45,418 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:40:45,418 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:45,418 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/af8c91dde83f490ea7805844475ddeff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6e2db26d703249c4aa5c30749f2c8a98, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ff48429dfb594dc0a8a7df1e39030084] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=35.3 K 2024-11-09T22:40:45,418 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 84eef6ee590e47c2a74e250473c94a10, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731192041898 2024-11-09T22:40:45,419 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting af8c91dde83f490ea7805844475ddeff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731192041898 2024-11-09T22:40:45,419 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c58dae23bf014ed8a95987e91a86d83d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731192042571 2024-11-09T22:40:45,419 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e2db26d703249c4aa5c30749f2c8a98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731192042571 2024-11-09T22:40:45,419 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a6512a6dc045278ce352f25bdc3255, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731192043761 2024-11-09T22:40:45,420 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff48429dfb594dc0a8a7df1e39030084, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731192043761 2024-11-09T22:40:45,445 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:45,445 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/c7d90b6150e54ed2847259a7f4889bb7 is 50, key is test_row_0/B:col10/1731192044906/Put/seqid=0 2024-11-09T22:40:45,448 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:45,449 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/541ca23da8e848c9811819dc29a500b1 is 50, key is test_row_0/A:col10/1731192044906/Put/seqid=0 2024-11-09T22:40:45,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742067_1243 (size=12207) 2024-11-09T22:40:45,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T22:40:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:45,485 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T22:40:45,485 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/c7d90b6150e54ed2847259a7f4889bb7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c7d90b6150e54ed2847259a7f4889bb7 2024-11-09T22:40:45,496 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into c7d90b6150e54ed2847259a7f4889bb7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:45,496 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:45,496 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192045416; duration=0sec 2024-11-09T22:40:45,500 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:45,501 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:40:45,501 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c37232f5562742f2bc55d8fd2790c616 is 50, key is test_row_0/A:col10/1731192044934/Put/seqid=0 2024-11-09T22:40:45,509 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:45,509 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/C is initiating minor compaction (all files) 2024-11-09T22:40:45,509 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/C in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:45,510 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ebecd7d6f5e444caa149eedac6568857, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9080dad29eae4d18a30a0e32370fc308, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e65cefb2da2247fc8bd56d4690418e27] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=35.3 K 2024-11-09T22:40:45,510 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebecd7d6f5e444caa149eedac6568857, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731192041898 2024-11-09T22:40:45,511 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9080dad29eae4d18a30a0e32370fc308, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731192042571 2024-11-09T22:40:45,511 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65cefb2da2247fc8bd56d4690418e27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731192043761 2024-11-09T22:40:45,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742068_1244 (size=12207) 2024-11-09T22:40:45,546 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/541ca23da8e848c9811819dc29a500b1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/541ca23da8e848c9811819dc29a500b1 2024-11-09T22:40:45,553 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 541ca23da8e848c9811819dc29a500b1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:45,553 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:45,553 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:45,554 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192045415; duration=0sec 2024-11-09T22:40:45,554 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:45,554 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:40:45,554 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ea23a631d90c44afb647b3b432b7ed9f is 50, key is test_row_0/C:col10/1731192044906/Put/seqid=0 2024-11-09T22:40:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:45,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:45,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742069_1245 (size=12001) 2024-11-09T22:40:45,570 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c37232f5562742f2bc55d8fd2790c616 2024-11-09T22:40:45,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192105585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192105587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192105588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192105590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742070_1246 (size=12207) 2024-11-09T22:40:45,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d7f7ece324b248b586a7bbee903d0f8d is 50, key is test_row_0/B:col10/1731192044934/Put/seqid=0 2024-11-09T22:40:45,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742071_1247 (size=12001) 2024-11-09T22:40:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T22:40:45,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192105695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192105705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192105705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192105705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192105905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192105915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192105915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192105915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,004 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ea23a631d90c44afb647b3b432b7ed9f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ea23a631d90c44afb647b3b432b7ed9f 2024-11-09T22:40:46,008 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/C of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into ea23a631d90c44afb647b3b432b7ed9f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:46,009 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:46,009 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, priority=13, startTime=1731192045416; duration=0sec 2024-11-09T22:40:46,009 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:46,009 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:40:46,053 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d7f7ece324b248b586a7bbee903d0f8d 2024-11-09T22:40:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/07f027fb29a04f81a8f154ebbc59e2bf is 50, key is test_row_0/C:col10/1731192044934/Put/seqid=0 2024-11-09T22:40:46,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742072_1248 (size=12001) 2024-11-09T22:40:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T22:40:46,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192106215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192106225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192106225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192106225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,469 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/07f027fb29a04f81a8f154ebbc59e2bf 2024-11-09T22:40:46,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c37232f5562742f2bc55d8fd2790c616 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c37232f5562742f2bc55d8fd2790c616 2024-11-09T22:40:46,478 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c37232f5562742f2bc55d8fd2790c616, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T22:40:46,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d7f7ece324b248b586a7bbee903d0f8d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d7f7ece324b248b586a7bbee903d0f8d 2024-11-09T22:40:46,482 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d7f7ece324b248b586a7bbee903d0f8d, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T22:40:46,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/07f027fb29a04f81a8f154ebbc59e2bf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/07f027fb29a04f81a8f154ebbc59e2bf 2024-11-09T22:40:46,487 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/07f027fb29a04f81a8f154ebbc59e2bf, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T22:40:46,487 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1002ms, sequenceid=112, compaction requested=false 2024-11-09T22:40:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:46,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-09T22:40:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-09T22:40:46,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-09T22:40:46,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4650 sec 2024-11-09T22:40:46,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4700 sec 2024-11-09T22:40:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:46,727 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:46,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T22:40:46,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/e412dde8300d44cba3320456b5fa3234 is 50, key is test_row_0/A:col10/1731192046726/Put/seqid=0 2024-11-09T22:40:46,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742073_1249 (size=14491) 2024-11-09T22:40:46,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/e412dde8300d44cba3320456b5fa3234 2024-11-09T22:40:46,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3384a91a845940c6a08c8c0e927d8206 is 50, key is test_row_0/B:col10/1731192046726/Put/seqid=0 2024-11-09T22:40:46,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742074_1250 (size=12151) 2024-11-09T22:40:46,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192106750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192106752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192106753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192106754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192106865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192106865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192106865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192106865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192107075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192107075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192107075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192107075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3384a91a845940c6a08c8c0e927d8206 2024-11-09T22:40:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T22:40:47,176 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:47,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:47,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/0a7a8449a0f44304b71f3b1c8c99c8f1 is 50, key is test_row_0/C:col10/1731192046726/Put/seqid=0 2024-11-09T22:40:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-09T22:40:47,178 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T22:40:47,179 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:47,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:47,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742075_1251 (size=12151) 2024-11-09T22:40:47,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/0a7a8449a0f44304b71f3b1c8c99c8f1 2024-11-09T22:40:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/e412dde8300d44cba3320456b5fa3234 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e412dde8300d44cba3320456b5fa3234 2024-11-09T22:40:47,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e412dde8300d44cba3320456b5fa3234, entries=200, sequenceid=133, filesize=14.2 K 2024-11-09T22:40:47,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3384a91a845940c6a08c8c0e927d8206 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3384a91a845940c6a08c8c0e927d8206 2024-11-09T22:40:47,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3384a91a845940c6a08c8c0e927d8206, entries=150, sequenceid=133, filesize=11.9 K 2024-11-09T22:40:47,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/0a7a8449a0f44304b71f3b1c8c99c8f1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/0a7a8449a0f44304b71f3b1c8c99c8f1 2024-11-09T22:40:47,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/0a7a8449a0f44304b71f3b1c8c99c8f1, entries=150, sequenceid=133, filesize=11.9 K 2024-11-09T22:40:47,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 524ms, sequenceid=133, compaction requested=true 2024-11-09T22:40:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:47,251 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:47,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:47,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:47,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:47,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:47,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:47,253 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:47,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:47,255 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:47,255 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:40:47,255 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,255 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/541ca23da8e848c9811819dc29a500b1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c37232f5562742f2bc55d8fd2790c616, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e412dde8300d44cba3320456b5fa3234] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=37.8 K 2024-11-09T22:40:47,255 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:47,255 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:40:47,255 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,256 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c7d90b6150e54ed2847259a7f4889bb7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d7f7ece324b248b586a7bbee903d0f8d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3384a91a845940c6a08c8c0e927d8206] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=35.5 K 2024-11-09T22:40:47,256 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d90b6150e54ed2847259a7f4889bb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731192043761 2024-11-09T22:40:47,256 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 541ca23da8e848c9811819dc29a500b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731192043761 2024-11-09T22:40:47,257 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f7ece324b248b586a7bbee903d0f8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731192044928 2024-11-09T22:40:47,257 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c37232f5562742f2bc55d8fd2790c616, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731192044928 2024-11-09T22:40:47,257 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3384a91a845940c6a08c8c0e927d8206, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731192045586 2024-11-09T22:40:47,258 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e412dde8300d44cba3320456b5fa3234, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731192045585 2024-11-09T22:40:47,278 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:47,279 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/e4809338fac14ce8ab6369f0c52a69af is 50, key is test_row_0/B:col10/1731192046726/Put/seqid=0 2024-11-09T22:40:47,285 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T22:40:47,285 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/cc470c7dbfbf41ea8ecb19bea291d412 is 50, key is test_row_0/A:col10/1731192046726/Put/seqid=0 2024-11-09T22:40:47,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742076_1252 (size=12459) 2024-11-09T22:40:47,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T22:40:47,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,333 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T22:40:47,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742077_1253 (size=12459) 2024-11-09T22:40:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/cea4c5baac68400fbea49c01e90704d2 is 50, key is test_row_0/A:col10/1731192046738/Put/seqid=0 2024-11-09T22:40:47,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742078_1254 (size=12151) 2024-11-09T22:40:47,351 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/cea4c5baac68400fbea49c01e90704d2 2024-11-09T22:40:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3d855284f05b44c4a22e5add07f781b7 is 50, key is test_row_0/B:col10/1731192046738/Put/seqid=0 2024-11-09T22:40:47,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742079_1255 (size=12151) 2024-11-09T22:40:47,397 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3d855284f05b44c4a22e5add07f781b7 2024-11-09T22:40:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:47,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:47,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/4d675507890846c1b9444d11fe514e3a is 50, key is test_row_0/C:col10/1731192046738/Put/seqid=0 2024-11-09T22:40:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192107426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192107423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192107431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192107431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742080_1256 (size=12151) 2024-11-09T22:40:47,444 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/4d675507890846c1b9444d11fe514e3a 2024-11-09T22:40:47,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/cea4c5baac68400fbea49c01e90704d2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cea4c5baac68400fbea49c01e90704d2 2024-11-09T22:40:47,456 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cea4c5baac68400fbea49c01e90704d2, entries=150, sequenceid=151, filesize=11.9 K 2024-11-09T22:40:47,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3d855284f05b44c4a22e5add07f781b7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3d855284f05b44c4a22e5add07f781b7 2024-11-09T22:40:47,462 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3d855284f05b44c4a22e5add07f781b7, entries=150, sequenceid=151, filesize=11.9 K 2024-11-09T22:40:47,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/4d675507890846c1b9444d11fe514e3a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/4d675507890846c1b9444d11fe514e3a 2024-11-09T22:40:47,468 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/4d675507890846c1b9444d11fe514e3a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-09T22:40:47,469 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 136ms, sequenceid=151, compaction requested=true 2024-11-09T22:40:47,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:47,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-09T22:40:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-09T22:40:47,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-09T22:40:47,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 291 msec 2024-11-09T22:40:47,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 297 msec 2024-11-09T22:40:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T22:40:47,495 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:47,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-09T22:40:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T22:40:47,498 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:47,499 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:47,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:47,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T22:40:47,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/9632b685b1ee4525af46abf3d7f106ec is 50, key is test_row_0/A:col10/1731192047540/Put/seqid=0 2024-11-09T22:40:47,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192107566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192107568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192107567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192107568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742081_1257 (size=14541) 2024-11-09T22:40:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T22:40:47,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T22:40:47,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:47,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192107685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192107685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192107686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192107686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,726 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/e4809338fac14ce8ab6369f0c52a69af as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/e4809338fac14ce8ab6369f0c52a69af 2024-11-09T22:40:47,735 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into e4809338fac14ce8ab6369f0c52a69af(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:47,735 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:47,735 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192047253; duration=0sec 2024-11-09T22:40:47,735 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:47,736 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:40:47,736 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:40:47,737 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:40:47,737 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/C is initiating minor compaction (all files) 2024-11-09T22:40:47,737 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/C in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,739 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ea23a631d90c44afb647b3b432b7ed9f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/07f027fb29a04f81a8f154ebbc59e2bf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/0a7a8449a0f44304b71f3b1c8c99c8f1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/4d675507890846c1b9444d11fe514e3a] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=47.4 K 2024-11-09T22:40:47,740 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ea23a631d90c44afb647b3b432b7ed9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731192043761 2024-11-09T22:40:47,740 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 07f027fb29a04f81a8f154ebbc59e2bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731192044928 2024-11-09T22:40:47,741 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7a8449a0f44304b71f3b1c8c99c8f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731192045586 2024-11-09T22:40:47,741 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d675507890846c1b9444d11fe514e3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731192046738 2024-11-09T22:40:47,745 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/cc470c7dbfbf41ea8ecb19bea291d412 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cc470c7dbfbf41ea8ecb19bea291d412 2024-11-09T22:40:47,752 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into cc470c7dbfbf41ea8ecb19bea291d412(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:47,752 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:47,752 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192047251; duration=0sec 2024-11-09T22:40:47,752 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:47,752 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:40:47,757 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#C#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:47,757 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/d53d7d1159744fe1899a143aee5cc0b4 is 50, key is test_row_0/C:col10/1731192046738/Put/seqid=0 2024-11-09T22:40:47,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742082_1258 (size=12493) 2024-11-09T22:40:47,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T22:40:47,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:47,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T22:40:47,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192107896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192107896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192107896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192107896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:47,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T22:40:47,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:47,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:47,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:47,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/9632b685b1ee4525af46abf3d7f106ec 2024-11-09T22:40:47,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3e4896e965ae42c4b2305db24ddcf800 is 50, key is test_row_0/B:col10/1731192047540/Put/seqid=0 2024-11-09T22:40:47,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742083_1259 (size=12151) 2024-11-09T22:40:47,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3e4896e965ae42c4b2305db24ddcf800 2024-11-09T22:40:48,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b40bf0294cab4c1782ea872b39f3fc29 is 50, key is test_row_0/C:col10/1731192047540/Put/seqid=0 2024-11-09T22:40:48,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742084_1260 (size=12151) 2024-11-09T22:40:48,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b40bf0294cab4c1782ea872b39f3fc29 2024-11-09T22:40:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/9632b685b1ee4525af46abf3d7f106ec as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/9632b685b1ee4525af46abf3d7f106ec 2024-11-09T22:40:48,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/9632b685b1ee4525af46abf3d7f106ec, entries=200, sequenceid=172, filesize=14.2 K 2024-11-09T22:40:48,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3e4896e965ae42c4b2305db24ddcf800 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3e4896e965ae42c4b2305db24ddcf800 2024-11-09T22:40:48,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3e4896e965ae42c4b2305db24ddcf800, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T22:40:48,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b40bf0294cab4c1782ea872b39f3fc29 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b40bf0294cab4c1782ea872b39f3fc29 2024-11-09T22:40:48,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b40bf0294cab4c1782ea872b39f3fc29, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T22:40:48,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 514ms, sequenceid=172, compaction requested=true 2024-11-09T22:40:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:48,057 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:48,058 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:48,058 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:40:48,058 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:48,058 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cc470c7dbfbf41ea8ecb19bea291d412, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cea4c5baac68400fbea49c01e90704d2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/9632b685b1ee4525af46abf3d7f106ec] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=38.2 K 2024-11-09T22:40:48,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:48,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:48,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:48,059 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc470c7dbfbf41ea8ecb19bea291d412, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731192045586 2024-11-09T22:40:48,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:48,059 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting cea4c5baac68400fbea49c01e90704d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731192046738 2024-11-09T22:40:48,060 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9632b685b1ee4525af46abf3d7f106ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192047429 2024-11-09T22:40:48,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:48,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T22:40:48,073 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:48,073 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/2ea98a1d20b84221825ea042901202a7 is 50, key is test_row_0/A:col10/1731192047540/Put/seqid=0 2024-11-09T22:40:48,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742085_1261 (size=12561) 2024-11-09T22:40:48,085 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/2ea98a1d20b84221825ea042901202a7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ea98a1d20b84221825ea042901202a7 2024-11-09T22:40:48,091 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 2ea98a1d20b84221825ea042901202a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:48,091 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:48,091 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192048057; duration=0sec 2024-11-09T22:40:48,091 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T22:40:48,091 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:40:48,091 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-09T22:40:48,092 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T22:40:48,092 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T22:40:48,092 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. because compaction request was cancelled 2024-11-09T22:40:48,092 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:40:48,092 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:48,093 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:48,093 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:40:48,093 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:48,093 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/e4809338fac14ce8ab6369f0c52a69af, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3d855284f05b44c4a22e5add07f781b7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3e4896e965ae42c4b2305db24ddcf800] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=35.9 K 2024-11-09T22:40:48,094 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4809338fac14ce8ab6369f0c52a69af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731192045586 2024-11-09T22:40:48,094 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d855284f05b44c4a22e5add07f781b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731192046738 2024-11-09T22:40:48,094 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4896e965ae42c4b2305db24ddcf800, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192047429 2024-11-09T22:40:48,101 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:48,102 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/a181bb8298c7417d8208e8fd1a99c842 is 50, key is test_row_0/B:col10/1731192047540/Put/seqid=0 2024-11-09T22:40:48,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742086_1262 (size=12561) 2024-11-09T22:40:48,112 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/a181bb8298c7417d8208e8fd1a99c842 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/a181bb8298c7417d8208e8fd1a99c842 2024-11-09T22:40:48,118 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into a181bb8298c7417d8208e8fd1a99c842(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:48,118 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:48,118 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192048059; duration=0sec 2024-11-09T22:40:48,118 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:48,118 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:40:48,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T22:40:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:48,123 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T22:40:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T22:40:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/2ff54670604a4b33b5c115e7ec297517 is 50, key is test_row_0/A:col10/1731192047566/Put/seqid=0 2024-11-09T22:40:48,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742087_1263 (size=12151) 2024-11-09T22:40:48,131 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/2ff54670604a4b33b5c115e7ec297517 2024-11-09T22:40:48,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/cb51c70c44da49c29d22b28993370eb9 is 50, key is test_row_0/B:col10/1731192047566/Put/seqid=0 2024-11-09T22:40:48,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742088_1264 (size=12151) 2024-11-09T22:40:48,174 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/d53d7d1159744fe1899a143aee5cc0b4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/d53d7d1159744fe1899a143aee5cc0b4 2024-11-09T22:40:48,178 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/C of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into d53d7d1159744fe1899a143aee5cc0b4(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:48,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:48,178 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, priority=12, startTime=1731192047253; duration=0sec 2024-11-09T22:40:48,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:48,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:40:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:48,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:48,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192108275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192108276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192108277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192108277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192108384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192108385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192108385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192108385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,556 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/cb51c70c44da49c29d22b28993370eb9 2024-11-09T22:40:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/5a42f08ecfce4093be10fc523575d7a7 is 50, key is test_row_0/C:col10/1731192047566/Put/seqid=0 2024-11-09T22:40:48,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742089_1265 (size=12151) 2024-11-09T22:40:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192108595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192108595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192108595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192108596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T22:40:48,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192108904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192108904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192108904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192108905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:48,982 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/5a42f08ecfce4093be10fc523575d7a7 2024-11-09T22:40:48,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/2ff54670604a4b33b5c115e7ec297517 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ff54670604a4b33b5c115e7ec297517 2024-11-09T22:40:48,990 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ff54670604a4b33b5c115e7ec297517, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T22:40:48,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/cb51c70c44da49c29d22b28993370eb9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/cb51c70c44da49c29d22b28993370eb9 2024-11-09T22:40:48,994 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/cb51c70c44da49c29d22b28993370eb9, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T22:40:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192108995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:48,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/5a42f08ecfce4093be10fc523575d7a7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/5a42f08ecfce4093be10fc523575d7a7 2024-11-09T22:40:48,995 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 3a78d83b1d743c9a9aaf5a1e71b1bc87 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:40:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,000 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/5a42f08ecfce4093be10fc523575d7a7, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T22:40:49,001 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 878ms, sequenceid=191, compaction requested=true 2024-11-09T22:40:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-09T22:40:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-09T22:40:49,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-09T22:40:49,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5040 sec 2024-11-09T22:40:49,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5090 sec 2024-11-09T22:40:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:49,417 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:49,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T22:40:49,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c3c735b3f9974c4b870aebe1e7457b45 is 50, key is test_row_0/A:col10/1731192049415/Put/seqid=0 2024-11-09T22:40:49,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742090_1266 (size=12151) 2024-11-09T22:40:49,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192109427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192109430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192109431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192109431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192109535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192109544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192109544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192109544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T22:40:49,645 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:49,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-09T22:40:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T22:40:49,647 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:49,648 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:49,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:49,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192109745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T22:40:49,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192109754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192109754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192109755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:49,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T22:40:49,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:49,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:49,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:49,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:49,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:49,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c3c735b3f9974c4b870aebe1e7457b45 2024-11-09T22:40:49,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/99ca1e9ef0e04ef7b179ca7c77f028b6 is 50, key is test_row_0/B:col10/1731192049415/Put/seqid=0 2024-11-09T22:40:49,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742091_1267 (size=12151) 2024-11-09T22:40:49,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T22:40:49,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:49,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:49,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:49,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:49,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T22:40:50,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192110055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192110065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:50,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192110065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192110065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:50,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T22:40:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/99ca1e9ef0e04ef7b179ca7c77f028b6 2024-11-09T22:40:50,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/f7d26a6984ed4a518277bbac43374b81 is 50, key is test_row_0/C:col10/1731192049415/Put/seqid=0 2024-11-09T22:40:50,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742092_1268 (size=12151) 2024-11-09T22:40:50,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T22:40:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T22:40:50,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T22:40:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T22:40:50,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:50,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:50,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192110565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:50,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192110575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:50,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192110575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:50,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192110575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:50,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/f7d26a6984ed4a518277bbac43374b81 2024-11-09T22:40:50,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c3c735b3f9974c4b870aebe1e7457b45 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c3c735b3f9974c4b870aebe1e7457b45 2024-11-09T22:40:50,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c3c735b3f9974c4b870aebe1e7457b45, entries=150, sequenceid=216, filesize=11.9 K 2024-11-09T22:40:50,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/99ca1e9ef0e04ef7b179ca7c77f028b6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/99ca1e9ef0e04ef7b179ca7c77f028b6 2024-11-09T22:40:50,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/99ca1e9ef0e04ef7b179ca7c77f028b6, entries=150, sequenceid=216, filesize=11.9 K 2024-11-09T22:40:50,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/f7d26a6984ed4a518277bbac43374b81 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/f7d26a6984ed4a518277bbac43374b81 2024-11-09T22:40:50,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/f7d26a6984ed4a518277bbac43374b81, entries=150, sequenceid=216, filesize=11.9 K 2024-11-09T22:40:50,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1255ms, sequenceid=216, compaction requested=true 2024-11-09T22:40:50,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:50,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:50,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:50,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:50,673 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:50,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:50,673 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:50,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:50,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:50,674 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:50,674 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:50,674 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:40:50,674 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:40:50,674 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,674 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,674 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ea98a1d20b84221825ea042901202a7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ff54670604a4b33b5c115e7ec297517, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c3c735b3f9974c4b870aebe1e7457b45] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.0 K 2024-11-09T22:40:50,674 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/a181bb8298c7417d8208e8fd1a99c842, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/cb51c70c44da49c29d22b28993370eb9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/99ca1e9ef0e04ef7b179ca7c77f028b6] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.0 K 2024-11-09T22:40:50,674 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting a181bb8298c7417d8208e8fd1a99c842, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192047429 2024-11-09T22:40:50,674 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea98a1d20b84221825ea042901202a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192047429 2024-11-09T22:40:50,674 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting cb51c70c44da49c29d22b28993370eb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731192047557 2024-11-09T22:40:50,674 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff54670604a4b33b5c115e7ec297517, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731192047557 2024-11-09T22:40:50,675 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ca1e9ef0e04ef7b179ca7c77f028b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731192048275 2024-11-09T22:40:50,675 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c735b3f9974c4b870aebe1e7457b45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731192048275 2024-11-09T22:40:50,681 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:50,681 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:50,681 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/bbe7e257b63b48bba182f8a09dc89ab6 is 50, key is test_row_0/A:col10/1731192049415/Put/seqid=0 2024-11-09T22:40:50,681 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d237151ed6e3498ca83ff2f166220a23 is 50, key is test_row_0/B:col10/1731192049415/Put/seqid=0 2024-11-09T22:40:50,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742093_1269 (size=12663) 2024-11-09T22:40:50,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742094_1270 (size=12663) 2024-11-09T22:40:50,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T22:40:50,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:50,717 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T22:40:50,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/52127059aee8496c8d24a8189519f807 is 50, key is test_row_0/A:col10/1731192049430/Put/seqid=0 2024-11-09T22:40:50,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742095_1271 (size=12151) 2024-11-09T22:40:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T22:40:51,100 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/bbe7e257b63b48bba182f8a09dc89ab6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bbe7e257b63b48bba182f8a09dc89ab6 2024-11-09T22:40:51,101 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d237151ed6e3498ca83ff2f166220a23 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d237151ed6e3498ca83ff2f166220a23 2024-11-09T22:40:51,104 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into bbe7e257b63b48bba182f8a09dc89ab6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:51,104 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:51,104 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192050673; duration=0sec 2024-11-09T22:40:51,104 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:51,105 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:40:51,105 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:40:51,105 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into d237151ed6e3498ca83ff2f166220a23(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:51,105 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:51,105 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192050673; duration=0sec 2024-11-09T22:40:51,105 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:51,105 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:40:51,106 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:40:51,107 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/C is initiating minor compaction (all files) 2024-11-09T22:40:51,107 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/C in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:51,107 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/d53d7d1159744fe1899a143aee5cc0b4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b40bf0294cab4c1782ea872b39f3fc29, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/5a42f08ecfce4093be10fc523575d7a7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/f7d26a6984ed4a518277bbac43374b81] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=47.8 K 2024-11-09T22:40:51,107 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d53d7d1159744fe1899a143aee5cc0b4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731192046738 2024-11-09T22:40:51,107 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40bf0294cab4c1782ea872b39f3fc29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192047429 2024-11-09T22:40:51,108 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a42f08ecfce4093be10fc523575d7a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731192047557 2024-11-09T22:40:51,108 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7d26a6984ed4a518277bbac43374b81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731192048275 2024-11-09T22:40:51,116 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:51,117 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/16f9b43acf6c4f76851a58c088050550 is 50, key is test_row_0/C:col10/1731192049415/Put/seqid=0 2024-11-09T22:40:51,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/52127059aee8496c8d24a8189519f807 2024-11-09T22:40:51,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742096_1272 (size=12629) 2024-11-09T22:40:51,135 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/16f9b43acf6c4f76851a58c088050550 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/16f9b43acf6c4f76851a58c088050550 2024-11-09T22:40:51,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/85febd77fce04e4f93b65c2f0bc6d315 is 50, key is test_row_0/B:col10/1731192049430/Put/seqid=0 2024-11-09T22:40:51,144 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/C of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 16f9b43acf6c4f76851a58c088050550(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:51,144 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:51,144 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, priority=12, startTime=1731192050673; duration=0sec 2024-11-09T22:40:51,144 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:51,144 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:40:51,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742097_1273 (size=12151) 2024-11-09T22:40:51,548 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/85febd77fce04e4f93b65c2f0bc6d315 2024-11-09T22:40:51,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b18c7bd323374e6fa3660d8a21f2edae is 50, key is test_row_0/C:col10/1731192049430/Put/seqid=0 2024-11-09T22:40:51,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742098_1274 (size=12151) 2024-11-09T22:40:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:51,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192111601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192111604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192111605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192111606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192111715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192111715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192111715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192111715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T22:40:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192111925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192111925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192111925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192111925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:51,960 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b18c7bd323374e6fa3660d8a21f2edae 2024-11-09T22:40:51,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/52127059aee8496c8d24a8189519f807 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/52127059aee8496c8d24a8189519f807 2024-11-09T22:40:51,969 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/52127059aee8496c8d24a8189519f807, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T22:40:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/85febd77fce04e4f93b65c2f0bc6d315 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/85febd77fce04e4f93b65c2f0bc6d315 2024-11-09T22:40:51,973 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/85febd77fce04e4f93b65c2f0bc6d315, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T22:40:51,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b18c7bd323374e6fa3660d8a21f2edae as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b18c7bd323374e6fa3660d8a21f2edae 2024-11-09T22:40:51,980 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b18c7bd323374e6fa3660d8a21f2edae, entries=150, sequenceid=229, filesize=11.9 K 2024-11-09T22:40:51,981 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1263ms, sequenceid=229, compaction requested=false 2024-11-09T22:40:51,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:51,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:51,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-09T22:40:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-09T22:40:51,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-09T22:40:51,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3340 sec 2024-11-09T22:40:51,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.3390 sec 2024-11-09T22:40:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:52,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T22:40:52,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/bf42c8bf68d842b6a2540ac89b46aa13 is 50, key is test_row_0/A:col10/1731192051604/Put/seqid=0 2024-11-09T22:40:52,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742099_1275 (size=12151) 2024-11-09T22:40:52,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192112248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192112247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192112247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192112252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192112355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192112355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192112355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192112365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192112564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192112565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192112565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192112575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/bf42c8bf68d842b6a2540ac89b46aa13 2024-11-09T22:40:52,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/dfc62eb819aa44c8b4aae8f8d1737886 is 50, key is test_row_0/B:col10/1731192051604/Put/seqid=0 2024-11-09T22:40:52,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742100_1276 (size=12151) 2024-11-09T22:40:52,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192112877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192112877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192112877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:52,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192112885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:53,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/dfc62eb819aa44c8b4aae8f8d1737886 2024-11-09T22:40:53,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/872522f717024f209bc7fa76767a7ef6 is 50, key is test_row_0/C:col10/1731192051604/Put/seqid=0 2024-11-09T22:40:53,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742101_1277 (size=12151) 2024-11-09T22:40:53,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192113385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:53,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192113385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192113394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192113395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:53,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/872522f717024f209bc7fa76767a7ef6 2024-11-09T22:40:53,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/bf42c8bf68d842b6a2540ac89b46aa13 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bf42c8bf68d842b6a2540ac89b46aa13 2024-11-09T22:40:53,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bf42c8bf68d842b6a2540ac89b46aa13, entries=150, sequenceid=256, filesize=11.9 K 2024-11-09T22:40:53,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/dfc62eb819aa44c8b4aae8f8d1737886 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/dfc62eb819aa44c8b4aae8f8d1737886 2024-11-09T22:40:53,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/dfc62eb819aa44c8b4aae8f8d1737886, entries=150, sequenceid=256, filesize=11.9 K 2024-11-09T22:40:53,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/872522f717024f209bc7fa76767a7ef6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/872522f717024f209bc7fa76767a7ef6 2024-11-09T22:40:53,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/872522f717024f209bc7fa76767a7ef6, entries=150, sequenceid=256, filesize=11.9 K 2024-11-09T22:40:53,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1267ms, sequenceid=256, compaction requested=true 2024-11-09T22:40:53,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:53,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:53,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:53,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:53,506 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:53,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:53,506 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:53,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:53,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:53,507 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:53,507 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:53,507 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:40:53,507 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:53,507 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d237151ed6e3498ca83ff2f166220a23, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/85febd77fce04e4f93b65c2f0bc6d315, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/dfc62eb819aa44c8b4aae8f8d1737886] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.1 K 2024-11-09T22:40:53,507 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:40:53,507 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:53,507 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bbe7e257b63b48bba182f8a09dc89ab6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/52127059aee8496c8d24a8189519f807, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bf42c8bf68d842b6a2540ac89b46aa13] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.1 K 2024-11-09T22:40:53,508 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d237151ed6e3498ca83ff2f166220a23, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731192048275 2024-11-09T22:40:53,508 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe7e257b63b48bba182f8a09dc89ab6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731192048275 2024-11-09T22:40:53,509 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 85febd77fce04e4f93b65c2f0bc6d315, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731192049425 2024-11-09T22:40:53,509 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52127059aee8496c8d24a8189519f807, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731192049425 2024-11-09T22:40:53,509 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc62eb819aa44c8b4aae8f8d1737886, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731192051600 2024-11-09T22:40:53,509 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf42c8bf68d842b6a2540ac89b46aa13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731192051600 2024-11-09T22:40:53,518 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:53,519 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/929683a4556c40c0a42e9c2e6040b96a is 50, key is test_row_0/B:col10/1731192051604/Put/seqid=0 2024-11-09T22:40:53,522 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:53,523 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/5b20f9157a4a4543b5c5d4d4a71c1a2e is 50, key is test_row_0/A:col10/1731192051604/Put/seqid=0 2024-11-09T22:40:53,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742102_1278 (size=12765) 2024-11-09T22:40:53,547 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/929683a4556c40c0a42e9c2e6040b96a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/929683a4556c40c0a42e9c2e6040b96a 2024-11-09T22:40:53,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742103_1279 (size=12765) 2024-11-09T22:40:53,560 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 929683a4556c40c0a42e9c2e6040b96a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:53,560 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:53,560 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192053506; duration=0sec 2024-11-09T22:40:53,560 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:53,560 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:40:53,560 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:53,562 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:53,562 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/C is initiating minor compaction (all files) 2024-11-09T22:40:53,563 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/C in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:53,563 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/16f9b43acf6c4f76851a58c088050550, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b18c7bd323374e6fa3660d8a21f2edae, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/872522f717024f209bc7fa76767a7ef6] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.1 K 2024-11-09T22:40:53,564 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f9b43acf6c4f76851a58c088050550, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731192048275 2024-11-09T22:40:53,565 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting b18c7bd323374e6fa3660d8a21f2edae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731192049425 2024-11-09T22:40:53,565 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 872522f717024f209bc7fa76767a7ef6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731192051600 2024-11-09T22:40:53,581 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#C#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:53,582 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/e5fa0ef3ac4a4361bd7d13dd2f61a6e1 is 50, key is test_row_0/C:col10/1731192051604/Put/seqid=0 2024-11-09T22:40:53,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742104_1280 (size=12731) 2024-11-09T22:40:53,601 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/e5fa0ef3ac4a4361bd7d13dd2f61a6e1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e5fa0ef3ac4a4361bd7d13dd2f61a6e1 2024-11-09T22:40:53,612 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/C of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into e5fa0ef3ac4a4361bd7d13dd2f61a6e1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:53,612 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:53,612 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, priority=13, startTime=1731192053506; duration=0sec 2024-11-09T22:40:53,612 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:53,612 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:40:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T22:40:53,835 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:53,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-09T22:40:53,837 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T22:40:53,838 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:53,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T22:40:53,963 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/5b20f9157a4a4543b5c5d4d4a71c1a2e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5b20f9157a4a4543b5c5d4d4a71c1a2e 2024-11-09T22:40:53,968 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 5b20f9157a4a4543b5c5d4d4a71c1a2e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:53,968 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:53,968 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192053505; duration=0sec 2024-11-09T22:40:53,968 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:53,968 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:40:53,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T22:40:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:53,990 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:40:53,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/62045038386e4e5b83a02cc72ae4d246 is 50, key is test_row_0/A:col10/1731192052246/Put/seqid=0 2024-11-09T22:40:54,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742105_1281 (size=12301) 2024-11-09T22:40:54,000 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/62045038386e4e5b83a02cc72ae4d246 2024-11-09T22:40:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/36b45539c99a41ecb79d63833d718472 is 50, key is test_row_0/B:col10/1731192052246/Put/seqid=0 2024-11-09T22:40:54,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742106_1282 (size=12301) 2024-11-09T22:40:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T22:40:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:54,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:54,426 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/36b45539c99a41ecb79d63833d718472 2024-11-09T22:40:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192114430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192114431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/7821ecc63fdf407085ba792cc1954a19 is 50, key is test_row_0/C:col10/1731192052246/Put/seqid=0 2024-11-09T22:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192114433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192114433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742107_1283 (size=12301) 2024-11-09T22:40:54,441 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/7821ecc63fdf407085ba792cc1954a19 2024-11-09T22:40:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/62045038386e4e5b83a02cc72ae4d246 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/62045038386e4e5b83a02cc72ae4d246 2024-11-09T22:40:54,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/62045038386e4e5b83a02cc72ae4d246, entries=150, sequenceid=271, filesize=12.0 K 2024-11-09T22:40:54,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/36b45539c99a41ecb79d63833d718472 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/36b45539c99a41ecb79d63833d718472 2024-11-09T22:40:54,459 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/36b45539c99a41ecb79d63833d718472, entries=150, sequenceid=271, filesize=12.0 K 2024-11-09T22:40:54,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/7821ecc63fdf407085ba792cc1954a19 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7821ecc63fdf407085ba792cc1954a19 2024-11-09T22:40:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T22:40:54,466 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7821ecc63fdf407085ba792cc1954a19, entries=150, sequenceid=271, filesize=12.0 K 2024-11-09T22:40:54,467 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 477ms, sequenceid=271, compaction requested=false 2024-11-09T22:40:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-09T22:40:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-09T22:40:54,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-09T22:40:54,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 631 msec 2024-11-09T22:40:54,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 636 msec 2024-11-09T22:40:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:54,548 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:54,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:40:54,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/dd81c95e38ec4033859c29d4043bbca1 is 50, key is test_row_0/A:col10/1731192054548/Put/seqid=0 2024-11-09T22:40:54,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742108_1284 (size=12301) 2024-11-09T22:40:54,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/dd81c95e38ec4033859c29d4043bbca1 2024-11-09T22:40:54,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192114557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192114557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192114563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/8f16316ed2334abc9088828a86c9b360 is 50, key is test_row_0/B:col10/1731192054548/Put/seqid=0 2024-11-09T22:40:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192114564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742109_1285 (size=12301) 2024-11-09T22:40:54,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192114664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192114665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192114675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192114675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192114874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192114875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192114885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192114885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:54,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/8f16316ed2334abc9088828a86c9b360 2024-11-09T22:40:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T22:40:54,975 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:54,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-09T22:40:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T22:40:54,977 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:54,978 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:54,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:54,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/55586f612d3a4bc09c25c3f7ff9e8964 is 50, key is test_row_0/C:col10/1731192054548/Put/seqid=0 2024-11-09T22:40:54,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742110_1286 (size=12301) 2024-11-09T22:40:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T22:40:55,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T22:40:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192115184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192115185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192115195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192115195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T22:40:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T22:40:55,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/55586f612d3a4bc09c25c3f7ff9e8964 2024-11-09T22:40:55,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/dd81c95e38ec4033859c29d4043bbca1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/dd81c95e38ec4033859c29d4043bbca1 2024-11-09T22:40:55,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/dd81c95e38ec4033859c29d4043bbca1, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T22:40:55,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/8f16316ed2334abc9088828a86c9b360 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/8f16316ed2334abc9088828a86c9b360 2024-11-09T22:40:55,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/8f16316ed2334abc9088828a86c9b360, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T22:40:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/55586f612d3a4bc09c25c3f7ff9e8964 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/55586f612d3a4bc09c25c3f7ff9e8964 2024-11-09T22:40:55,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/55586f612d3a4bc09c25c3f7ff9e8964, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T22:40:55,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T22:40:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:55,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 909ms, sequenceid=297, compaction requested=true 2024-11-09T22:40:55,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:55,457 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:55,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:55,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:55,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:55,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:55,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:55,458 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:55,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:55,458 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:55,458 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:40:55,458 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:55,458 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,459 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:40:55,459 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,459 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5b20f9157a4a4543b5c5d4d4a71c1a2e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/62045038386e4e5b83a02cc72ae4d246, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/dd81c95e38ec4033859c29d4043bbca1] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.5 K 2024-11-09T22:40:55,459 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/929683a4556c40c0a42e9c2e6040b96a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/36b45539c99a41ecb79d63833d718472, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/8f16316ed2334abc9088828a86c9b360] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.5 K 2024-11-09T22:40:55,459 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 929683a4556c40c0a42e9c2e6040b96a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731192051600 2024-11-09T22:40:55,459 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b20f9157a4a4543b5c5d4d4a71c1a2e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731192051600 2024-11-09T22:40:55,459 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62045038386e4e5b83a02cc72ae4d246, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731192052246 2024-11-09T22:40:55,459 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b45539c99a41ecb79d63833d718472, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731192052246 2024-11-09T22:40:55,459 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd81c95e38ec4033859c29d4043bbca1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731192054424 2024-11-09T22:40:55,459 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f16316ed2334abc9088828a86c9b360, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731192054424 2024-11-09T22:40:55,465 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:55,466 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:55,466 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/c0c014eb6e54446ba8f223f48b223c5c is 50, key is test_row_0/B:col10/1731192054548/Put/seqid=0 2024-11-09T22:40:55,466 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/5730506f305a49298845f1c2ec806e13 is 50, key is test_row_0/A:col10/1731192054548/Put/seqid=0 2024-11-09T22:40:55,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742112_1288 (size=13017) 2024-11-09T22:40:55,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742111_1287 (size=13017) 2024-11-09T22:40:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T22:40:55,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T22:40:55,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,608 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:40:55,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c12efdeea75b4b17b934b9a9c4420fc2 is 50, key is test_row_0/A:col10/1731192054560/Put/seqid=0 2024-11-09T22:40:55,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742113_1289 (size=12301) 2024-11-09T22:40:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:55,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192115722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192115726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192115727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192115727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192115834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192115834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192115834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192115835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:55,874 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/c0c014eb6e54446ba8f223f48b223c5c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c0c014eb6e54446ba8f223f48b223c5c 2024-11-09T22:40:55,874 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/5730506f305a49298845f1c2ec806e13 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5730506f305a49298845f1c2ec806e13 2024-11-09T22:40:55,880 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into c0c014eb6e54446ba8f223f48b223c5c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:55,880 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 5730506f305a49298845f1c2ec806e13(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:55,880 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:55,880 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:55,880 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192055457; duration=0sec 2024-11-09T22:40:55,880 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192055458; duration=0sec 2024-11-09T22:40:55,880 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:55,880 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:40:55,880 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:55,880 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:40:55,880 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:55,882 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:55,882 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/C is initiating minor compaction (all files) 2024-11-09T22:40:55,882 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/C in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:55,882 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e5fa0ef3ac4a4361bd7d13dd2f61a6e1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7821ecc63fdf407085ba792cc1954a19, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/55586f612d3a4bc09c25c3f7ff9e8964] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.5 K 2024-11-09T22:40:55,883 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting e5fa0ef3ac4a4361bd7d13dd2f61a6e1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731192051600 2024-11-09T22:40:55,883 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7821ecc63fdf407085ba792cc1954a19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731192052246 2024-11-09T22:40:55,883 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 55586f612d3a4bc09c25c3f7ff9e8964, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731192054424 2024-11-09T22:40:55,889 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:55,890 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/44562fb2b3024ad4bd1c61611410e465 is 50, key is test_row_0/C:col10/1731192054548/Put/seqid=0 2024-11-09T22:40:55,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742114_1290 (size=12983) 2024-11-09T22:40:56,016 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c12efdeea75b4b17b934b9a9c4420fc2 2024-11-09T22:40:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d006da0c4b8e4dc89ae5588f0d76ff51 is 50, key is test_row_0/B:col10/1731192054560/Put/seqid=0 2024-11-09T22:40:56,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742115_1291 (size=12301) 2024-11-09T22:40:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192116045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192116045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192116045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192116045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T22:40:56,297 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/44562fb2b3024ad4bd1c61611410e465 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/44562fb2b3024ad4bd1c61611410e465 2024-11-09T22:40:56,302 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/C of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 44562fb2b3024ad4bd1c61611410e465(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:56,302 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:56,302 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, priority=13, startTime=1731192055458; duration=0sec 2024-11-09T22:40:56,302 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:56,302 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:40:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192116355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192116355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192116355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192116355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,426 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d006da0c4b8e4dc89ae5588f0d76ff51 2024-11-09T22:40:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/bcf3bc656f294b4e827a293ccd98a5b8 is 50, key is test_row_0/C:col10/1731192054560/Put/seqid=0 2024-11-09T22:40:56,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742116_1292 (size=12301) 2024-11-09T22:40:56,837 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/bcf3bc656f294b4e827a293ccd98a5b8 2024-11-09T22:40:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/c12efdeea75b4b17b934b9a9c4420fc2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c12efdeea75b4b17b934b9a9c4420fc2 2024-11-09T22:40:56,845 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c12efdeea75b4b17b934b9a9c4420fc2, entries=150, sequenceid=308, filesize=12.0 K 2024-11-09T22:40:56,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d006da0c4b8e4dc89ae5588f0d76ff51 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d006da0c4b8e4dc89ae5588f0d76ff51 2024-11-09T22:40:56,849 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d006da0c4b8e4dc89ae5588f0d76ff51, entries=150, sequenceid=308, filesize=12.0 K 2024-11-09T22:40:56,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/bcf3bc656f294b4e827a293ccd98a5b8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/bcf3bc656f294b4e827a293ccd98a5b8 2024-11-09T22:40:56,854 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/bcf3bc656f294b4e827a293ccd98a5b8, entries=150, sequenceid=308, filesize=12.0 K 2024-11-09T22:40:56,855 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1247ms, sequenceid=308, compaction requested=false 2024-11-09T22:40:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-09T22:40:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-09T22:40:56,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-09T22:40:56,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-09T22:40:56,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.8830 sec 2024-11-09T22:40:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:56,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:56,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T22:40:56,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/aec6bb8ce2d443299d0db3e8f45ad559 is 50, key is test_row_0/A:col10/1731192056869/Put/seqid=0 2024-11-09T22:40:56,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742117_1293 (size=12301) 2024-11-09T22:40:56,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192116875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192116875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192116877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192116877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192116985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192116985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192116985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192116985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T22:40:57,125 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:57,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-09T22:40:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T22:40:57,127 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:57,127 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:57,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:57,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192117195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192117195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192117196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192117196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T22:40:57,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/aec6bb8ce2d443299d0db3e8f45ad559 2024-11-09T22:40:57,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T22:40:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/ceaa4578c5d94feaa321227eac0d0c7d is 50, key is test_row_0/B:col10/1731192056869/Put/seqid=0 2024-11-09T22:40:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742118_1294 (size=12301) 2024-11-09T22:40:57,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T22:40:57,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:57,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T22:40:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192117504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192117505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192117505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192117505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:57,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T22:40:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/ceaa4578c5d94feaa321227eac0d0c7d 2024-11-09T22:40:57,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/6f341cffdf3b41ce9c32ddfc4b081091 is 50, key is test_row_0/C:col10/1731192056869/Put/seqid=0 2024-11-09T22:40:57,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742119_1295 (size=12301) 2024-11-09T22:40:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T22:40:57,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T22:40:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T22:40:57,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:57,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:57,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192118015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:58,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192118015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:58,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192118015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:58,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192118016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:58,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T22:40:58,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:58,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:58,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:58,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:58,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:58,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/6f341cffdf3b41ce9c32ddfc4b081091 2024-11-09T22:40:58,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/aec6bb8ce2d443299d0db3e8f45ad559 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/aec6bb8ce2d443299d0db3e8f45ad559 2024-11-09T22:40:58,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/aec6bb8ce2d443299d0db3e8f45ad559, entries=150, sequenceid=340, filesize=12.0 K 2024-11-09T22:40:58,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/ceaa4578c5d94feaa321227eac0d0c7d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ceaa4578c5d94feaa321227eac0d0c7d 2024-11-09T22:40:58,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ceaa4578c5d94feaa321227eac0d0c7d, entries=150, sequenceid=340, filesize=12.0 K 2024-11-09T22:40:58,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/6f341cffdf3b41ce9c32ddfc4b081091 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/6f341cffdf3b41ce9c32ddfc4b081091 2024-11-09T22:40:58,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/6f341cffdf3b41ce9c32ddfc4b081091, entries=150, sequenceid=340, filesize=12.0 K 2024-11-09T22:40:58,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1306ms, sequenceid=340, compaction requested=true 2024-11-09T22:40:58,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:58,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:40:58,175 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:58,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:58,175 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:58,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:40:58,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:58,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:40:58,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:58,176 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:58,177 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:40:58,177 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:58,177 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5730506f305a49298845f1c2ec806e13, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c12efdeea75b4b17b934b9a9c4420fc2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/aec6bb8ce2d443299d0db3e8f45ad559] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.7 K 2024-11-09T22:40:58,177 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:58,177 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:40:58,177 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:58,177 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c0c014eb6e54446ba8f223f48b223c5c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d006da0c4b8e4dc89ae5588f0d76ff51, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ceaa4578c5d94feaa321227eac0d0c7d] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.7 K 2024-11-09T22:40:58,177 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5730506f305a49298845f1c2ec806e13, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731192054424 2024-11-09T22:40:58,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c014eb6e54446ba8f223f48b223c5c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731192054424 2024-11-09T22:40:58,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d006da0c4b8e4dc89ae5588f0d76ff51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731192054556 2024-11-09T22:40:58,178 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c12efdeea75b4b17b934b9a9c4420fc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731192054556 2024-11-09T22:40:58,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ceaa4578c5d94feaa321227eac0d0c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731192055726 2024-11-09T22:40:58,179 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec6bb8ce2d443299d0db3e8f45ad559, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731192055726 2024-11-09T22:40:58,207 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:58,207 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/e68c4a56fb4142d18398093064fa3455 is 50, key is test_row_0/A:col10/1731192056869/Put/seqid=0 2024-11-09T22:40:58,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T22:40:58,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:58,225 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T22:40:58,231 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:58,231 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d531c60ba94f4e08b323ee20f6050d0b is 50, key is test_row_0/B:col10/1731192056869/Put/seqid=0 2024-11-09T22:40:58,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/32d007a8674342f489c2ab28407920ca is 50, key is test_row_2/A:col10/1731192056875/Put/seqid=0 2024-11-09T22:40:58,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742120_1296 (size=13119) 2024-11-09T22:40:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T22:40:58,268 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/e68c4a56fb4142d18398093064fa3455 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e68c4a56fb4142d18398093064fa3455 2024-11-09T22:40:58,276 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into e68c4a56fb4142d18398093064fa3455(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:58,276 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:58,276 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192058175; duration=0sec 2024-11-09T22:40:58,276 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:40:58,276 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:40:58,276 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:40:58,278 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:40:58,278 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/C is initiating minor compaction (all files) 2024-11-09T22:40:58,278 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/C in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:58,278 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/44562fb2b3024ad4bd1c61611410e465, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/bcf3bc656f294b4e827a293ccd98a5b8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/6f341cffdf3b41ce9c32ddfc4b081091] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=36.7 K 2024-11-09T22:40:58,278 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44562fb2b3024ad4bd1c61611410e465, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731192054424 2024-11-09T22:40:58,279 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcf3bc656f294b4e827a293ccd98a5b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731192054556 2024-11-09T22:40:58,279 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f341cffdf3b41ce9c32ddfc4b081091, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731192055726 2024-11-09T22:40:58,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742121_1297 (size=13119) 2024-11-09T22:40:58,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742122_1298 (size=7415) 2024-11-09T22:40:58,324 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/32d007a8674342f489c2ab28407920ca 2024-11-09T22:40:58,325 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:40:58,326 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/611340520f964067b3c6864aaaf34152 is 50, key is test_row_0/C:col10/1731192056869/Put/seqid=0 2024-11-09T22:40:58,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3f890a95da3b452794bbb0b60b7d79ff is 50, key is test_row_2/B:col10/1731192056875/Put/seqid=0 2024-11-09T22:40:58,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742123_1299 (size=13085) 2024-11-09T22:40:58,368 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/611340520f964067b3c6864aaaf34152 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/611340520f964067b3c6864aaaf34152 2024-11-09T22:40:58,373 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/C of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 611340520f964067b3c6864aaaf34152(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:58,374 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:58,374 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, priority=13, startTime=1731192058176; duration=0sec 2024-11-09T22:40:58,374 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:58,374 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:40:58,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742124_1300 (size=7415) 2024-11-09T22:40:58,695 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/d531c60ba94f4e08b323ee20f6050d0b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d531c60ba94f4e08b323ee20f6050d0b 2024-11-09T22:40:58,701 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into d531c60ba94f4e08b323ee20f6050d0b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:40:58,702 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:58,702 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192058175; duration=0sec 2024-11-09T22:40:58,702 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:40:58,702 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:40:58,776 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3f890a95da3b452794bbb0b60b7d79ff 2024-11-09T22:40:58,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/846a51ae362745ecaa39c6a3991fa56b is 50, key is test_row_2/C:col10/1731192056875/Put/seqid=0 2024-11-09T22:40:58,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742125_1301 (size=7415) 2024-11-09T22:40:59,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:59,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192119043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192119044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192119046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192119047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192119047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192119155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192119156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192119156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192119156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192119156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,191 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/846a51ae362745ecaa39c6a3991fa56b 2024-11-09T22:40:59,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/32d007a8674342f489c2ab28407920ca as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/32d007a8674342f489c2ab28407920ca 2024-11-09T22:40:59,199 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/32d007a8674342f489c2ab28407920ca, entries=50, sequenceid=347, filesize=7.2 K 2024-11-09T22:40:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/3f890a95da3b452794bbb0b60b7d79ff as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3f890a95da3b452794bbb0b60b7d79ff 2024-11-09T22:40:59,205 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3f890a95da3b452794bbb0b60b7d79ff, entries=50, sequenceid=347, filesize=7.2 K 2024-11-09T22:40:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/846a51ae362745ecaa39c6a3991fa56b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/846a51ae362745ecaa39c6a3991fa56b 2024-11-09T22:40:59,215 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/846a51ae362745ecaa39c6a3991fa56b, entries=50, sequenceid=347, filesize=7.2 K 2024-11-09T22:40:59,217 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 991ms, sequenceid=347, compaction requested=false 2024-11-09T22:40:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:40:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-09T22:40:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-09T22:40:59,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-09T22:40:59,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0930 sec 2024-11-09T22:40:59,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.0980 sec 2024-11-09T22:40:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T22:40:59,275 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:40:59,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:40:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-09T22:40:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T22:40:59,278 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:40:59,279 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:40:59,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:40:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:40:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:40:59,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-09T22:40:59,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192119367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192119369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/7612715891504411ac2d59072c211486 is 50, key is test_row_0/A:col10/1731192059367/Put/seqid=0 2024-11-09T22:40:59,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192119369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192119370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192119374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742126_1302 (size=12301) 2024-11-09T22:40:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T22:40:59,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:40:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192119475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192119475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192119484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192119485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192119485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:40:59,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:59,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T22:40:59,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192119684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192119684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192119695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192119695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192119695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:40:59,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:59,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/7612715891504411ac2d59072c211486 2024-11-09T22:40:59,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/be4d604de0e44baa9795110fbab0c933 is 50, key is test_row_0/B:col10/1731192059367/Put/seqid=0 2024-11-09T22:40:59,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742127_1303 (size=12301) 2024-11-09T22:40:59,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:40:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:40:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:40:59,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:40:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T22:40:59,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:40:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192119995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192119995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192120004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:00,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192120004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192120004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:41:00,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:41:00,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/be4d604de0e44baa9795110fbab0c933 2024-11-09T22:41:00,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:41:00,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:41:00,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6 is 50, key is test_row_0/C:col10/1731192059367/Put/seqid=0 2024-11-09T22:41:00,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742128_1304 (size=12301) 2024-11-09T22:41:00,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:41:00,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:41:00,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T22:41:00,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:41:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. as already flushing 2024-11-09T22:41:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:00,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60014 deadline: 1731192120505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:00,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60080 deadline: 1731192120505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:00,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60064 deadline: 1731192120514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60096 deadline: 1731192120514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60104 deadline: 1731192120514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:00,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6 2024-11-09T22:41:00,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/7612715891504411ac2d59072c211486 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/7612715891504411ac2d59072c211486 2024-11-09T22:41:00,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/7612715891504411ac2d59072c211486, entries=150, sequenceid=382, filesize=12.0 K 2024-11-09T22:41:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/be4d604de0e44baa9795110fbab0c933 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/be4d604de0e44baa9795110fbab0c933 2024-11-09T22:41:00,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/be4d604de0e44baa9795110fbab0c933, entries=150, sequenceid=382, filesize=12.0 K 2024-11-09T22:41:00,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6 2024-11-09T22:41:00,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6, entries=150, sequenceid=382, filesize=12.0 K 2024-11-09T22:41:00,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1254ms, sequenceid=382, compaction requested=true 2024-11-09T22:41:00,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:41:00,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:00,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:00,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:00,621 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:00,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:00,621 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:00,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3a78d83b1d743c9a9aaf5a1e71b1bc87:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:00,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:00,624 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:00,624 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:00,624 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/B is initiating minor compaction (all files) 2024-11-09T22:41:00,624 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/A is initiating minor compaction (all files) 2024-11-09T22:41:00,624 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/B in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,624 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/A in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,624 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d531c60ba94f4e08b323ee20f6050d0b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3f890a95da3b452794bbb0b60b7d79ff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/be4d604de0e44baa9795110fbab0c933] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=32.1 K 2024-11-09T22:41:00,624 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e68c4a56fb4142d18398093064fa3455, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/32d007a8674342f489c2ab28407920ca, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/7612715891504411ac2d59072c211486] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=32.1 K 2024-11-09T22:41:00,624 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e68c4a56fb4142d18398093064fa3455, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731192055726 2024-11-09T22:41:00,624 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d531c60ba94f4e08b323ee20f6050d0b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731192055726 2024-11-09T22:41:00,625 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f890a95da3b452794bbb0b60b7d79ff, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731192056875 2024-11-09T22:41:00,625 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d007a8674342f489c2ab28407920ca, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731192056875 2024-11-09T22:41:00,625 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7612715891504411ac2d59072c211486, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731192059366 2024-11-09T22:41:00,625 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting be4d604de0e44baa9795110fbab0c933, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731192059366 2024-11-09T22:41:00,630 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:00,631 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:00,631 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/38215d0200c9481e8ece7b944156cc3a is 50, key is test_row_0/B:col10/1731192059367/Put/seqid=0 2024-11-09T22:41:00,631 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/a6b178cbd9494b2eb2d259f40b4ab50f is 50, key is test_row_0/A:col10/1731192059367/Put/seqid=0 2024-11-09T22:41:00,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742130_1306 (size=13221) 2024-11-09T22:41:00,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742129_1305 (size=13221) 2024-11-09T22:41:00,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-09T22:41:00,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:00,653 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-09T22:41:00,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/5488582164a545be9170a90b03456b57 is 50, key is test_row_1/A:col10/1731192059369/Put/seqid=0 2024-11-09T22:41:00,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742131_1307 (size=7415) 2024-11-09T22:41:00,661 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/5488582164a545be9170a90b03456b57 2024-11-09T22:41:00,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/11a226686a9a4065b2f72b01cb2b07b1 is 50, key is test_row_1/B:col10/1731192059369/Put/seqid=0 2024-11-09T22:41:00,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742132_1308 (size=7415) 2024-11-09T22:41:00,709 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-11-09T22:41:00,709 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:00,709 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,709 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:00,710 INFO [Thread-1063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1063. 2024-11-09T22:41:00,710 DEBUG [Thread-1063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:00,710 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,710 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:00,710 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-11-09T22:41:00,710 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:00,710 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,710 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:00,710 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-11-09T22:41:00,710 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:00,710 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,710 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:00,711 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-09T22:41:00,711 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:00,712 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,712 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:00,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:01,050 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/a6b178cbd9494b2eb2d259f40b4ab50f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/a6b178cbd9494b2eb2d259f40b4ab50f 2024-11-09T22:41:01,050 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/38215d0200c9481e8ece7b944156cc3a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/38215d0200c9481e8ece7b944156cc3a 2024-11-09T22:41:01,055 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/A of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into a6b178cbd9494b2eb2d259f40b4ab50f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:01,055 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/B of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 38215d0200c9481e8ece7b944156cc3a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:01,055 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:41:01,055 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:41:01,055 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/A, priority=13, startTime=1731192060621; duration=0sec 2024-11-09T22:41:01,055 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/B, priority=13, startTime=1731192060621; duration=0sec 2024-11-09T22:41:01,056 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:01,056 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:B 2024-11-09T22:41:01,056 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:01,056 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:A 2024-11-09T22:41:01,056 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:01,057 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:01,057 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 3a78d83b1d743c9a9aaf5a1e71b1bc87/C is initiating minor compaction (all files) 2024-11-09T22:41:01,057 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3a78d83b1d743c9a9aaf5a1e71b1bc87/C in TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:01,057 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/611340520f964067b3c6864aaaf34152, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/846a51ae362745ecaa39c6a3991fa56b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp, totalSize=32.0 K 2024-11-09T22:41:01,057 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 611340520f964067b3c6864aaaf34152, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731192055726 2024-11-09T22:41:01,057 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 846a51ae362745ecaa39c6a3991fa56b, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731192056875 2024-11-09T22:41:01,058 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting b5dc94c4b5e641cb9ab760ac7e4b7ba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731192059366 2024-11-09T22:41:01,063 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3a78d83b1d743c9a9aaf5a1e71b1bc87#C#compaction#267 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:01,064 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/59db2b3838df4349954b86e2cedbdbfc is 50, key is test_row_0/C:col10/1731192059367/Put/seqid=0 2024-11-09T22:41:01,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742133_1309 (size=13187) 2024-11-09T22:41:01,077 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/11a226686a9a4065b2f72b01cb2b07b1 2024-11-09T22:41:01,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ba880cca151f42cba51fea61147c9995 is 50, key is test_row_1/C:col10/1731192059369/Put/seqid=0 2024-11-09T22:41:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742134_1310 (size=7415) 2024-11-09T22:41:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T22:41:01,473 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/59db2b3838df4349954b86e2cedbdbfc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/59db2b3838df4349954b86e2cedbdbfc 2024-11-09T22:41:01,477 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3a78d83b1d743c9a9aaf5a1e71b1bc87/C of 3a78d83b1d743c9a9aaf5a1e71b1bc87 into 59db2b3838df4349954b86e2cedbdbfc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:01,477 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:41:01,477 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., storeName=3a78d83b1d743c9a9aaf5a1e71b1bc87/C, priority=13, startTime=1731192060621; duration=0sec 2024-11-09T22:41:01,478 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:01,478 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3a78d83b1d743c9a9aaf5a1e71b1bc87:C 2024-11-09T22:41:01,488 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ba880cca151f42cba51fea61147c9995 2024-11-09T22:41:01,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/5488582164a545be9170a90b03456b57 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5488582164a545be9170a90b03456b57 2024-11-09T22:41:01,496 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5488582164a545be9170a90b03456b57, entries=50, sequenceid=387, filesize=7.2 K 2024-11-09T22:41:01,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/11a226686a9a4065b2f72b01cb2b07b1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/11a226686a9a4065b2f72b01cb2b07b1 2024-11-09T22:41:01,500 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/11a226686a9a4065b2f72b01cb2b07b1, entries=50, sequenceid=387, filesize=7.2 K 2024-11-09T22:41:01,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/ba880cca151f42cba51fea61147c9995 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ba880cca151f42cba51fea61147c9995 2024-11-09T22:41:01,504 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ba880cca151f42cba51fea61147c9995, entries=50, sequenceid=387, filesize=7.2 K 2024-11-09T22:41:01,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 853ms, sequenceid=387, compaction requested=false 2024-11-09T22:41:01,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: 2024-11-09T22:41:01,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:01,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-09T22:41:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-09T22:41:01,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-09T22:41:01,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2270 sec 2024-11-09T22:41:01,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.2320 sec 2024-11-09T22:41:01,516 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-11-09T22:41:01,516 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-09T22:41:01,516 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:01,516 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,516 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:01,516 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,516 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,516 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:01,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:01,526 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-09T22:41:01,526 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-09T22:41:01,526 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:01,526 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:01,526 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,526 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,526 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,526 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:01,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:01,536 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-09T22:41:01,536 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:01,536 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,536 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:01,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T22:41:03,436 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4167 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3962 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3644 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4197 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3925 2024-11-09T22:41:03,436 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T22:41:03,436 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:41:03,436 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:41:03,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:03,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:03,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:03,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T22:41:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-09T22:41:03,440 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192063440"}]},"ts":"1731192063440"} 2024-11-09T22:41:03,441 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T22:41:03,441 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T22:41:03,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:41:03,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, UNASSIGN}] 2024-11-09T22:41:03,443 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, UNASSIGN 2024-11-09T22:41:03,444 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=3a78d83b1d743c9a9aaf5a1e71b1bc87, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:03,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, UNASSIGN because future has completed 2024-11-09T22:41:03,445 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:41:03,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:41:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-09T22:41:03,598 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:41:03,598 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:41:03,598 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 3a78d83b1d743c9a9aaf5a1e71b1bc87, disabling compactions & flushes 2024-11-09T22:41:03,598 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:03,598 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:03,598 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. after waiting 0 ms 2024-11-09T22:41:03,598 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:03,598 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T22:41:03,602 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/69da37a6fdb446168f645d971765b6bf is 50, key is test_row_0/A:col10/1731192061535/Put/seqid=0 2024-11-09T22:41:03,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742135_1311 (size=9857) 2024-11-09T22:41:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-09T22:41:04,008 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/69da37a6fdb446168f645d971765b6bf 2024-11-09T22:41:04,020 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/6982294251994461a5401b031ac073de is 50, key is test_row_0/B:col10/1731192061535/Put/seqid=0 2024-11-09T22:41:04,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742136_1312 (size=9857) 2024-11-09T22:41:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-09T22:41:04,426 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/6982294251994461a5401b031ac073de 2024-11-09T22:41:04,439 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/7a9f5e6f665647a9a11ece212c86bd11 is 50, key is test_row_0/C:col10/1731192061535/Put/seqid=0 2024-11-09T22:41:04,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742137_1313 (size=9857) 2024-11-09T22:41:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-09T22:41:04,845 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/7a9f5e6f665647a9a11ece212c86bd11 2024-11-09T22:41:04,858 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/A/69da37a6fdb446168f645d971765b6bf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/69da37a6fdb446168f645d971765b6bf 2024-11-09T22:41:04,863 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/69da37a6fdb446168f645d971765b6bf, entries=100, sequenceid=398, filesize=9.6 K 2024-11-09T22:41:04,864 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/B/6982294251994461a5401b031ac073de as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6982294251994461a5401b031ac073de 2024-11-09T22:41:04,869 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6982294251994461a5401b031ac073de, entries=100, sequenceid=398, filesize=9.6 K 2024-11-09T22:41:04,870 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/.tmp/C/7a9f5e6f665647a9a11ece212c86bd11 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7a9f5e6f665647a9a11ece212c86bd11 2024-11-09T22:41:04,873 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7a9f5e6f665647a9a11ece212c86bd11, entries=100, sequenceid=398, filesize=9.6 K 2024-11-09T22:41:04,874 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1276ms, sequenceid=398, compaction requested=true 2024-11-09T22:41:04,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/8eea8512a6f2475a803cb42d2ac321f2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/f969111511bc455eaafecf1dad41c11f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c6a12eb6e42e4d7f85170258caa1d355, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/84eef6ee590e47c2a74e250473c94a10, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c58dae23bf014ed8a95987e91a86d83d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e5a6512a6dc045278ce352f25bdc3255, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/541ca23da8e848c9811819dc29a500b1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c37232f5562742f2bc55d8fd2790c616, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e412dde8300d44cba3320456b5fa3234, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cc470c7dbfbf41ea8ecb19bea291d412, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cea4c5baac68400fbea49c01e90704d2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/9632b685b1ee4525af46abf3d7f106ec, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ea98a1d20b84221825ea042901202a7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ff54670604a4b33b5c115e7ec297517, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bbe7e257b63b48bba182f8a09dc89ab6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c3c735b3f9974c4b870aebe1e7457b45, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/52127059aee8496c8d24a8189519f807, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5b20f9157a4a4543b5c5d4d4a71c1a2e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bf42c8bf68d842b6a2540ac89b46aa13, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/62045038386e4e5b83a02cc72ae4d246, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5730506f305a49298845f1c2ec806e13, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/dd81c95e38ec4033859c29d4043bbca1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c12efdeea75b4b17b934b9a9c4420fc2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e68c4a56fb4142d18398093064fa3455, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/aec6bb8ce2d443299d0db3e8f45ad559, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/32d007a8674342f489c2ab28407920ca, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/7612715891504411ac2d59072c211486] to archive 2024-11-09T22:41:04,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:04,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/8eea8512a6f2475a803cb42d2ac321f2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/8eea8512a6f2475a803cb42d2ac321f2 2024-11-09T22:41:04,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/f969111511bc455eaafecf1dad41c11f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/f969111511bc455eaafecf1dad41c11f 2024-11-09T22:41:04,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c6a12eb6e42e4d7f85170258caa1d355 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c6a12eb6e42e4d7f85170258caa1d355 2024-11-09T22:41:04,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/84eef6ee590e47c2a74e250473c94a10 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/84eef6ee590e47c2a74e250473c94a10 2024-11-09T22:41:04,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c58dae23bf014ed8a95987e91a86d83d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c58dae23bf014ed8a95987e91a86d83d 2024-11-09T22:41:04,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e5a6512a6dc045278ce352f25bdc3255 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e5a6512a6dc045278ce352f25bdc3255 2024-11-09T22:41:04,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/541ca23da8e848c9811819dc29a500b1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/541ca23da8e848c9811819dc29a500b1 2024-11-09T22:41:04,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c37232f5562742f2bc55d8fd2790c616 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c37232f5562742f2bc55d8fd2790c616 2024-11-09T22:41:04,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e412dde8300d44cba3320456b5fa3234 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e412dde8300d44cba3320456b5fa3234 2024-11-09T22:41:04,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cc470c7dbfbf41ea8ecb19bea291d412 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cc470c7dbfbf41ea8ecb19bea291d412 2024-11-09T22:41:04,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cea4c5baac68400fbea49c01e90704d2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/cea4c5baac68400fbea49c01e90704d2 2024-11-09T22:41:04,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/9632b685b1ee4525af46abf3d7f106ec to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/9632b685b1ee4525af46abf3d7f106ec 2024-11-09T22:41:04,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ea98a1d20b84221825ea042901202a7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ea98a1d20b84221825ea042901202a7 2024-11-09T22:41:04,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ff54670604a4b33b5c115e7ec297517 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/2ff54670604a4b33b5c115e7ec297517 2024-11-09T22:41:04,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bbe7e257b63b48bba182f8a09dc89ab6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bbe7e257b63b48bba182f8a09dc89ab6 2024-11-09T22:41:04,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c3c735b3f9974c4b870aebe1e7457b45 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c3c735b3f9974c4b870aebe1e7457b45 2024-11-09T22:41:04,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/52127059aee8496c8d24a8189519f807 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/52127059aee8496c8d24a8189519f807 2024-11-09T22:41:04,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5b20f9157a4a4543b5c5d4d4a71c1a2e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5b20f9157a4a4543b5c5d4d4a71c1a2e 2024-11-09T22:41:04,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bf42c8bf68d842b6a2540ac89b46aa13 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/bf42c8bf68d842b6a2540ac89b46aa13 2024-11-09T22:41:04,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/62045038386e4e5b83a02cc72ae4d246 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/62045038386e4e5b83a02cc72ae4d246 2024-11-09T22:41:04,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5730506f305a49298845f1c2ec806e13 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5730506f305a49298845f1c2ec806e13 2024-11-09T22:41:04,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/dd81c95e38ec4033859c29d4043bbca1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/dd81c95e38ec4033859c29d4043bbca1 2024-11-09T22:41:04,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c12efdeea75b4b17b934b9a9c4420fc2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/c12efdeea75b4b17b934b9a9c4420fc2 2024-11-09T22:41:04,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e68c4a56fb4142d18398093064fa3455 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/e68c4a56fb4142d18398093064fa3455 2024-11-09T22:41:04,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/aec6bb8ce2d443299d0db3e8f45ad559 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/aec6bb8ce2d443299d0db3e8f45ad559 2024-11-09T22:41:04,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/32d007a8674342f489c2ab28407920ca to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/32d007a8674342f489c2ab28407920ca 2024-11-09T22:41:04,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/7612715891504411ac2d59072c211486 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/7612715891504411ac2d59072c211486 2024-11-09T22:41:04,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/21a6d6706970461eb4886f3ae33fd7f6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/fe6ca0b27f584a8988aed30d8f095898, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/af8c91dde83f490ea7805844475ddeff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/acb5fc40ea784271bd06ca35a48a5330, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6e2db26d703249c4aa5c30749f2c8a98, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c7d90b6150e54ed2847259a7f4889bb7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ff48429dfb594dc0a8a7df1e39030084, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d7f7ece324b248b586a7bbee903d0f8d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/e4809338fac14ce8ab6369f0c52a69af, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3384a91a845940c6a08c8c0e927d8206, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3d855284f05b44c4a22e5add07f781b7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/a181bb8298c7417d8208e8fd1a99c842, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3e4896e965ae42c4b2305db24ddcf800, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/cb51c70c44da49c29d22b28993370eb9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d237151ed6e3498ca83ff2f166220a23, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/99ca1e9ef0e04ef7b179ca7c77f028b6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/85febd77fce04e4f93b65c2f0bc6d315, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/929683a4556c40c0a42e9c2e6040b96a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/dfc62eb819aa44c8b4aae8f8d1737886, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/36b45539c99a41ecb79d63833d718472, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c0c014eb6e54446ba8f223f48b223c5c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/8f16316ed2334abc9088828a86c9b360, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d006da0c4b8e4dc89ae5588f0d76ff51, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d531c60ba94f4e08b323ee20f6050d0b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ceaa4578c5d94feaa321227eac0d0c7d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3f890a95da3b452794bbb0b60b7d79ff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/be4d604de0e44baa9795110fbab0c933] to archive 2024-11-09T22:41:04,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:04,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/21a6d6706970461eb4886f3ae33fd7f6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/21a6d6706970461eb4886f3ae33fd7f6 2024-11-09T22:41:04,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/fe6ca0b27f584a8988aed30d8f095898 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/fe6ca0b27f584a8988aed30d8f095898 2024-11-09T22:41:04,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/af8c91dde83f490ea7805844475ddeff to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/af8c91dde83f490ea7805844475ddeff 2024-11-09T22:41:04,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/acb5fc40ea784271bd06ca35a48a5330 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/acb5fc40ea784271bd06ca35a48a5330 2024-11-09T22:41:04,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6e2db26d703249c4aa5c30749f2c8a98 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6e2db26d703249c4aa5c30749f2c8a98 2024-11-09T22:41:04,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c7d90b6150e54ed2847259a7f4889bb7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c7d90b6150e54ed2847259a7f4889bb7 2024-11-09T22:41:04,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ff48429dfb594dc0a8a7df1e39030084 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ff48429dfb594dc0a8a7df1e39030084 2024-11-09T22:41:04,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d7f7ece324b248b586a7bbee903d0f8d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d7f7ece324b248b586a7bbee903d0f8d 2024-11-09T22:41:04,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/e4809338fac14ce8ab6369f0c52a69af to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/e4809338fac14ce8ab6369f0c52a69af 2024-11-09T22:41:04,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3384a91a845940c6a08c8c0e927d8206 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3384a91a845940c6a08c8c0e927d8206 2024-11-09T22:41:04,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3d855284f05b44c4a22e5add07f781b7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3d855284f05b44c4a22e5add07f781b7 2024-11-09T22:41:04,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/a181bb8298c7417d8208e8fd1a99c842 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/a181bb8298c7417d8208e8fd1a99c842 2024-11-09T22:41:04,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3e4896e965ae42c4b2305db24ddcf800 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3e4896e965ae42c4b2305db24ddcf800 2024-11-09T22:41:04,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/cb51c70c44da49c29d22b28993370eb9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/cb51c70c44da49c29d22b28993370eb9 2024-11-09T22:41:04,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d237151ed6e3498ca83ff2f166220a23 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d237151ed6e3498ca83ff2f166220a23 2024-11-09T22:41:04,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/99ca1e9ef0e04ef7b179ca7c77f028b6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/99ca1e9ef0e04ef7b179ca7c77f028b6 2024-11-09T22:41:04,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/85febd77fce04e4f93b65c2f0bc6d315 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/85febd77fce04e4f93b65c2f0bc6d315 2024-11-09T22:41:04,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/929683a4556c40c0a42e9c2e6040b96a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/929683a4556c40c0a42e9c2e6040b96a 2024-11-09T22:41:04,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/dfc62eb819aa44c8b4aae8f8d1737886 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/dfc62eb819aa44c8b4aae8f8d1737886 2024-11-09T22:41:04,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/36b45539c99a41ecb79d63833d718472 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/36b45539c99a41ecb79d63833d718472 2024-11-09T22:41:04,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c0c014eb6e54446ba8f223f48b223c5c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/c0c014eb6e54446ba8f223f48b223c5c 2024-11-09T22:41:04,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/8f16316ed2334abc9088828a86c9b360 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/8f16316ed2334abc9088828a86c9b360 2024-11-09T22:41:04,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d006da0c4b8e4dc89ae5588f0d76ff51 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d006da0c4b8e4dc89ae5588f0d76ff51 2024-11-09T22:41:04,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d531c60ba94f4e08b323ee20f6050d0b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/d531c60ba94f4e08b323ee20f6050d0b 2024-11-09T22:41:04,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ceaa4578c5d94feaa321227eac0d0c7d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/ceaa4578c5d94feaa321227eac0d0c7d 2024-11-09T22:41:04,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3f890a95da3b452794bbb0b60b7d79ff to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/3f890a95da3b452794bbb0b60b7d79ff 2024-11-09T22:41:04,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/be4d604de0e44baa9795110fbab0c933 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/be4d604de0e44baa9795110fbab0c933 2024-11-09T22:41:04,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ec4d7f35a0814e09ba3ab069a4dab030, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9ea7ff72c87b4c80843392391e4d6fd7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ebecd7d6f5e444caa149eedac6568857, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/34d17135e47742a99a3be89cc0ac6484, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9080dad29eae4d18a30a0e32370fc308, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ea23a631d90c44afb647b3b432b7ed9f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e65cefb2da2247fc8bd56d4690418e27, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/07f027fb29a04f81a8f154ebbc59e2bf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/0a7a8449a0f44304b71f3b1c8c99c8f1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/d53d7d1159744fe1899a143aee5cc0b4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/4d675507890846c1b9444d11fe514e3a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b40bf0294cab4c1782ea872b39f3fc29, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/5a42f08ecfce4093be10fc523575d7a7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/16f9b43acf6c4f76851a58c088050550, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/f7d26a6984ed4a518277bbac43374b81, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b18c7bd323374e6fa3660d8a21f2edae, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e5fa0ef3ac4a4361bd7d13dd2f61a6e1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/872522f717024f209bc7fa76767a7ef6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7821ecc63fdf407085ba792cc1954a19, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/44562fb2b3024ad4bd1c61611410e465, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/55586f612d3a4bc09c25c3f7ff9e8964, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/bcf3bc656f294b4e827a293ccd98a5b8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/611340520f964067b3c6864aaaf34152, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/6f341cffdf3b41ce9c32ddfc4b081091, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/846a51ae362745ecaa39c6a3991fa56b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6] to archive 2024-11-09T22:41:04,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:04,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ec4d7f35a0814e09ba3ab069a4dab030 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ec4d7f35a0814e09ba3ab069a4dab030 2024-11-09T22:41:04,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9ea7ff72c87b4c80843392391e4d6fd7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9ea7ff72c87b4c80843392391e4d6fd7 2024-11-09T22:41:04,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ebecd7d6f5e444caa149eedac6568857 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ebecd7d6f5e444caa149eedac6568857 2024-11-09T22:41:04,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/34d17135e47742a99a3be89cc0ac6484 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/34d17135e47742a99a3be89cc0ac6484 2024-11-09T22:41:04,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9080dad29eae4d18a30a0e32370fc308 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/9080dad29eae4d18a30a0e32370fc308 2024-11-09T22:41:04,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ea23a631d90c44afb647b3b432b7ed9f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ea23a631d90c44afb647b3b432b7ed9f 2024-11-09T22:41:04,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e65cefb2da2247fc8bd56d4690418e27 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e65cefb2da2247fc8bd56d4690418e27 2024-11-09T22:41:04,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/07f027fb29a04f81a8f154ebbc59e2bf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/07f027fb29a04f81a8f154ebbc59e2bf 2024-11-09T22:41:04,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/0a7a8449a0f44304b71f3b1c8c99c8f1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/0a7a8449a0f44304b71f3b1c8c99c8f1 2024-11-09T22:41:04,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/d53d7d1159744fe1899a143aee5cc0b4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/d53d7d1159744fe1899a143aee5cc0b4 2024-11-09T22:41:04,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/4d675507890846c1b9444d11fe514e3a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/4d675507890846c1b9444d11fe514e3a 2024-11-09T22:41:04,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b40bf0294cab4c1782ea872b39f3fc29 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b40bf0294cab4c1782ea872b39f3fc29 2024-11-09T22:41:04,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/5a42f08ecfce4093be10fc523575d7a7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/5a42f08ecfce4093be10fc523575d7a7 2024-11-09T22:41:04,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/16f9b43acf6c4f76851a58c088050550 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/16f9b43acf6c4f76851a58c088050550 2024-11-09T22:41:04,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/f7d26a6984ed4a518277bbac43374b81 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/f7d26a6984ed4a518277bbac43374b81 2024-11-09T22:41:04,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b18c7bd323374e6fa3660d8a21f2edae to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b18c7bd323374e6fa3660d8a21f2edae 2024-11-09T22:41:04,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e5fa0ef3ac4a4361bd7d13dd2f61a6e1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/e5fa0ef3ac4a4361bd7d13dd2f61a6e1 2024-11-09T22:41:04,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/872522f717024f209bc7fa76767a7ef6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/872522f717024f209bc7fa76767a7ef6 2024-11-09T22:41:04,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7821ecc63fdf407085ba792cc1954a19 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7821ecc63fdf407085ba792cc1954a19 2024-11-09T22:41:04,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/44562fb2b3024ad4bd1c61611410e465 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/44562fb2b3024ad4bd1c61611410e465 2024-11-09T22:41:04,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/55586f612d3a4bc09c25c3f7ff9e8964 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/55586f612d3a4bc09c25c3f7ff9e8964 2024-11-09T22:41:04,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/bcf3bc656f294b4e827a293ccd98a5b8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/bcf3bc656f294b4e827a293ccd98a5b8 2024-11-09T22:41:04,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/611340520f964067b3c6864aaaf34152 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/611340520f964067b3c6864aaaf34152 2024-11-09T22:41:04,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/6f341cffdf3b41ce9c32ddfc4b081091 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/6f341cffdf3b41ce9c32ddfc4b081091 2024-11-09T22:41:04,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/846a51ae362745ecaa39c6a3991fa56b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/846a51ae362745ecaa39c6a3991fa56b 2024-11-09T22:41:04,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/b5dc94c4b5e641cb9ab760ac7e4b7ba6 2024-11-09T22:41:04,956 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/recovered.edits/401.seqid, newMaxSeqId=401, maxSeqId=1 2024-11-09T22:41:04,956 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. 2024-11-09T22:41:04,956 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 3a78d83b1d743c9a9aaf5a1e71b1bc87: Waiting for close lock at 1731192063598Running coprocessor pre-close hooks at 1731192063598Disabling compacts and flushes for region at 1731192063598Disabling writes for close at 1731192063598Obtaining lock to block concurrent updates at 1731192063598Preparing flush snapshotting stores in 3a78d83b1d743c9a9aaf5a1e71b1bc87 at 1731192063598Finished memstore snapshotting TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731192063598Flushing stores of TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87. at 1731192063599 (+1 ms)Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/A: creating writer at 1731192063599Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/A: appending metadata at 1731192063601 (+2 ms)Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/A: closing flushed file at 1731192063602 (+1 ms)Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/B: creating writer at 1731192064017 (+415 ms)Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/B: appending metadata at 1731192064020 (+3 ms)Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/B: closing flushed file at 1731192064020Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/C: creating writer at 1731192064435 (+415 ms)Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/C: appending metadata at 1731192064439 (+4 ms)Flushing 3a78d83b1d743c9a9aaf5a1e71b1bc87/C: closing flushed file at 1731192064439Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a188012: reopening flushed file at 1731192064857 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ddf0966: reopening flushed file at 1731192064864 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b05e248: reopening flushed file at 1731192064869 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3a78d83b1d743c9a9aaf5a1e71b1bc87 in 1276ms, sequenceid=398, compaction requested=true at 1731192064874 (+5 ms)Writing region close event to WAL at 1731192064952 (+78 ms)Running coprocessor post-close hooks at 1731192064956 (+4 ms)Closed at 1731192064956 2024-11-09T22:41:04,958 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:41:04,959 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=3a78d83b1d743c9a9aaf5a1e71b1bc87, regionState=CLOSED 2024-11-09T22:41:04,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:41:04,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-09T22:41:04,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 3a78d83b1d743c9a9aaf5a1e71b1bc87, server=9d8c37854f6e,35101,1731191981027 in 1.5160 sec 2024-11-09T22:41:04,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-09T22:41:04,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3a78d83b1d743c9a9aaf5a1e71b1bc87, UNASSIGN in 1.5190 sec 2024-11-09T22:41:04,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-09T22:41:04,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-09T22:41:04,965 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192064965"}]},"ts":"1731192064965"} 2024-11-09T22:41:04,966 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T22:41:04,966 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T22:41:04,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5300 sec 2024-11-09T22:41:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-09T22:41:05,586 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:05,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T22:41:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:05,620 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-09T22:41:05,621 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:05,624 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:41:05,627 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/recovered.edits] 2024-11-09T22:41:05,629 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5488582164a545be9170a90b03456b57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/5488582164a545be9170a90b03456b57 2024-11-09T22:41:05,630 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/69da37a6fdb446168f645d971765b6bf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/69da37a6fdb446168f645d971765b6bf 2024-11-09T22:41:05,631 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/a6b178cbd9494b2eb2d259f40b4ab50f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/A/a6b178cbd9494b2eb2d259f40b4ab50f 2024-11-09T22:41:05,632 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/11a226686a9a4065b2f72b01cb2b07b1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/11a226686a9a4065b2f72b01cb2b07b1 2024-11-09T22:41:05,633 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/38215d0200c9481e8ece7b944156cc3a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/38215d0200c9481e8ece7b944156cc3a 2024-11-09T22:41:05,634 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6982294251994461a5401b031ac073de to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/B/6982294251994461a5401b031ac073de 2024-11-09T22:41:05,636 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/59db2b3838df4349954b86e2cedbdbfc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/59db2b3838df4349954b86e2cedbdbfc 2024-11-09T22:41:05,637 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7a9f5e6f665647a9a11ece212c86bd11 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/7a9f5e6f665647a9a11ece212c86bd11 2024-11-09T22:41:05,637 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ba880cca151f42cba51fea61147c9995 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/C/ba880cca151f42cba51fea61147c9995 2024-11-09T22:41:05,640 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/recovered.edits/401.seqid to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87/recovered.edits/401.seqid 2024-11-09T22:41:05,640 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/3a78d83b1d743c9a9aaf5a1e71b1bc87 2024-11-09T22:41:05,640 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T22:41:05,642 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:05,645 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T22:41:05,647 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T22:41:05,648 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:05,648 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T22:41:05,648 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731192065648"}]},"ts":"9223372036854775807"} 2024-11-09T22:41:05,650 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T22:41:05,650 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3a78d83b1d743c9a9aaf5a1e71b1bc87, NAME => 'TestAcidGuarantees,,1731192038511.3a78d83b1d743c9a9aaf5a1e71b1bc87.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T22:41:05,650 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T22:41:05,650 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731192065650"}]},"ts":"9223372036854775807"} 2024-11-09T22:41:05,652 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T22:41:05,653 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:05,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-09T22:41:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-09T22:41:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T22:41:05,725 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:05,735 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 244), OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=566 (was 597), ProcessCount=11 (was 11), AvailableMemoryMB=17677 (was 17928) 2024-11-09T22:41:05,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=566, ProcessCount=11, AvailableMemoryMB=17676 2024-11-09T22:41:05,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:41:05,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:41:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:05,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T22:41:05,748 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:05,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-09T22:41:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T22:41:05,749 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T22:41:05,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742138_1314 (size=959) 2024-11-09T22:41:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T22:41:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T22:41:06,161 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 2024-11-09T22:41:06,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742139_1315 (size=53) 2024-11-09T22:41:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T22:41:06,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:41:06,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1936e48909d1b634ac268f8f357af8e0, disabling compactions & flushes 2024-11-09T22:41:06,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:06,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:06,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. after waiting 0 ms 2024-11-09T22:41:06,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:06,573 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:06,573 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1936e48909d1b634ac268f8f357af8e0: Waiting for close lock at 1731192066572Disabling compacts and flushes for region at 1731192066572Disabling writes for close at 1731192066572Writing region close event to WAL at 1731192066572Closed at 1731192066573 (+1 ms) 2024-11-09T22:41:06,576 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T22:41:06,577 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731192066576"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731192066576"}]},"ts":"1731192066576"} 2024-11-09T22:41:06,581 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T22:41:06,582 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T22:41:06,583 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192066582"}]},"ts":"1731192066582"} 2024-11-09T22:41:06,584 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T22:41:06,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, ASSIGN}] 2024-11-09T22:41:06,585 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, ASSIGN 2024-11-09T22:41:06,586 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, ASSIGN; state=OFFLINE, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=false 2024-11-09T22:41:06,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=1936e48909d1b634ac268f8f357af8e0, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:06,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, ASSIGN because future has completed 2024-11-09T22:41:06,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:41:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T22:41:06,902 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:06,903 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:41:06,903 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,903 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:41:06,903 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,904 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,906 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,908 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1936e48909d1b634ac268f8f357af8e0 columnFamilyName A 2024-11-09T22:41:06,908 DEBUG [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:06,909 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(327): Store=1936e48909d1b634ac268f8f357af8e0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:06,909 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,911 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1936e48909d1b634ac268f8f357af8e0 columnFamilyName B 2024-11-09T22:41:06,911 DEBUG [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:06,912 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(327): Store=1936e48909d1b634ac268f8f357af8e0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:06,912 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,913 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1936e48909d1b634ac268f8f357af8e0 columnFamilyName C 2024-11-09T22:41:06,913 DEBUG [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:06,914 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(327): Store=1936e48909d1b634ac268f8f357af8e0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:06,914 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,915 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,916 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,917 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,917 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,918 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:41:06,919 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,921 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:41:06,922 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 1936e48909d1b634ac268f8f357af8e0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71942815, jitterRate=0.07203148305416107}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:41:06,922 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:06,923 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 1936e48909d1b634ac268f8f357af8e0: Running coprocessor pre-open hook at 1731192066904Writing region info on filesystem at 1731192066904Initializing all the Stores at 1731192066905 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192066905Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192066906 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192066906Cleaning up temporary data from old regions at 1731192066917 (+11 ms)Running coprocessor post-open hooks at 1731192066922 (+5 ms)Region opened successfully at 1731192066922 2024-11-09T22:41:06,923 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., pid=95, masterSystemTime=1731192066896 2024-11-09T22:41:06,925 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:06,925 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:06,925 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=1936e48909d1b634ac268f8f357af8e0, regionState=OPEN, openSeqNum=2, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:06,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:41:06,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-09T22:41:06,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 in 185 msec 2024-11-09T22:41:06,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-09T22:41:06,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, ASSIGN in 345 msec 2024-11-09T22:41:06,933 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T22:41:06,933 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192066933"}]},"ts":"1731192066933"} 2024-11-09T22:41:06,935 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T22:41:06,936 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T22:41:06,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1900 sec 2024-11-09T22:41:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-09T22:41:07,895 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:07,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:07,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:07,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:07,897 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7994aa7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:07,899 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@637b4afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:07,899 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:07,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:07,902 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T22:41:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d8c37854f6e,34651,1731191980160 2024-11-09T22:41:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5bd924ff 2024-11-09T22:41:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T22:41:07,904 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T22:41:07,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:41:07,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:41:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:07,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742140_1316 (size=995) 2024-11-09T22:41:08,321 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T22:41:08,321 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T22:41:08,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:41:08,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, REOPEN/MOVE}] 2024-11-09T22:41:08,329 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, REOPEN/MOVE 2024-11-09T22:41:08,330 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1936e48909d1b634ac268f8f357af8e0, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:08,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, REOPEN/MOVE because future has completed 2024-11-09T22:41:08,332 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:41:08,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:41:08,485 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,485 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:41:08,485 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 1936e48909d1b634ac268f8f357af8e0, disabling compactions & flushes 2024-11-09T22:41:08,485 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:08,486 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:08,486 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. after waiting 0 ms 2024-11-09T22:41:08,486 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:08,489 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T22:41:08,489 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:08,489 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 1936e48909d1b634ac268f8f357af8e0: Waiting for close lock at 1731192068485Running coprocessor pre-close hooks at 1731192068485Disabling compacts and flushes for region at 1731192068485Disabling writes for close at 1731192068486 (+1 ms)Writing region close event to WAL at 1731192068486Running coprocessor post-close hooks at 1731192068489 (+3 ms)Closed at 1731192068489 2024-11-09T22:41:08,489 WARN [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 1936e48909d1b634ac268f8f357af8e0 to self. 2024-11-09T22:41:08,491 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,492 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1936e48909d1b634ac268f8f357af8e0, regionState=CLOSED 2024-11-09T22:41:08,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:41:08,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-09T22:41:08,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 in 161 msec 2024-11-09T22:41:08,495 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, REOPEN/MOVE; state=CLOSED, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=true 2024-11-09T22:41:08,646 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1936e48909d1b634ac268f8f357af8e0, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:08,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, REOPEN/MOVE because future has completed 2024-11-09T22:41:08,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:41:08,807 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:08,807 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:41:08,808 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,808 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:41:08,808 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,808 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,810 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,811 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1936e48909d1b634ac268f8f357af8e0 columnFamilyName A 2024-11-09T22:41:08,813 DEBUG [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:08,813 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(327): Store=1936e48909d1b634ac268f8f357af8e0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:08,814 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,815 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1936e48909d1b634ac268f8f357af8e0 columnFamilyName B 2024-11-09T22:41:08,815 DEBUG [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:08,816 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(327): Store=1936e48909d1b634ac268f8f357af8e0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:08,816 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,817 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1936e48909d1b634ac268f8f357af8e0 columnFamilyName C 2024-11-09T22:41:08,817 DEBUG [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:08,818 INFO [StoreOpener-1936e48909d1b634ac268f8f357af8e0-1 {}] regionserver.HStore(327): Store=1936e48909d1b634ac268f8f357af8e0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:08,818 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,819 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,820 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,822 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,822 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,823 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:41:08,825 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,826 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 1936e48909d1b634ac268f8f357af8e0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73684477, jitterRate=0.09798426926136017}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:41:08,827 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,828 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 1936e48909d1b634ac268f8f357af8e0: Running coprocessor pre-open hook at 1731192068808Writing region info on filesystem at 1731192068808Initializing all the Stores at 1731192068809 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192068809Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192068810 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192068810Cleaning up temporary data from old regions at 1731192068822 (+12 ms)Running coprocessor post-open hooks at 1731192068827 (+5 ms)Region opened successfully at 1731192068828 (+1 ms) 2024-11-09T22:41:08,829 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., pid=100, masterSystemTime=1731192068802 2024-11-09T22:41:08,832 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:08,832 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:08,833 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1936e48909d1b634ac268f8f357af8e0, regionState=OPEN, openSeqNum=5, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:08,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:41:08,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-09T22:41:08,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 in 187 msec 2024-11-09T22:41:08,841 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-09T22:41:08,841 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, REOPEN/MOVE in 510 msec 2024-11-09T22:41:08,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-09T22:41:08,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-11-09T22:41:08,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-09T22:41:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-09T22:41:08,849 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:08,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,852 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe1b97a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,854 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5ebae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,856 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6282c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5c284d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260b91fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,862 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f4564c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,864 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7c8672, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375a7002, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,868 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a13019b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,869 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78718adf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,871 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5185c606, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,873 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31225469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,875 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a82b944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,878 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf99e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,879 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e694354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,881 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70843b85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,882 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4766c557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:08,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:08,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e62696a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:08,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd0c8b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:08,886 DEBUG [Thread-1465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,887 DEBUG [Thread-1467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,887 DEBUG [Thread-1452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,887 DEBUG [Thread-1450 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-09T22:41:08,888 DEBUG [Thread-1448 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,888 DEBUG [Thread-1459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,889 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:08,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,889 DEBUG [Thread-1461 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,889 DEBUG [Thread-1456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T22:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,890 DEBUG [Thread-1463 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,890 DEBUG [Thread-1454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,890 DEBUG [Thread-1459 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,891 DEBUG [Thread-1459 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,892 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:08,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:08,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:08,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:41:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:08,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:08,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:41:08,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fb79c52b3189485b830b630af61bf563_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192068903/Put/seqid=0 2024-11-09T22:41:08,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742141_1317 (size=19474) 2024-11-09T22:41:08,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192128960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:08,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192128968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:08,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192128969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:08,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192128970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192128971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T22:41:09,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:09,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:09,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192129075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192129075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192129084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192129084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192129084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T22:41:09,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T22:41:09,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192129285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192129285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192129294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192129294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192129295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,344 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:09,347 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fb79c52b3189485b830b630af61bf563_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb79c52b3189485b830b630af61bf563_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:09,348 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/5ddaafaf455d425abab7eed8e5cacf3d, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:09,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/5ddaafaf455d425abab7eed8e5cacf3d is 175, key is test_row_0/A:col10/1731192068903/Put/seqid=0 2024-11-09T22:41:09,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742142_1318 (size=56729) 2024-11-09T22:41:09,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:09,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:09,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:09,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:09,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T22:41:09,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192129595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192129595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192129605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192129605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192129605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:09,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:09,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:09,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/5ddaafaf455d425abab7eed8e5cacf3d 2024-11-09T22:41:09,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/49f1ae14f02d411caf1cc28cfa46c065 is 50, key is test_row_0/B:col10/1731192068903/Put/seqid=0 2024-11-09T22:41:09,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742143_1319 (size=9657) 2024-11-09T22:41:09,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:09,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:09,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:09,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:09,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:09,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T22:41:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192130105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192130105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:10,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:10,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:10,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192130115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192130115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:10,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192130124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:10,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/49f1ae14f02d411caf1cc28cfa46c065 2024-11-09T22:41:10,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/6dc14b97c6144487977d2778efefb66e is 50, key is test_row_0/C:col10/1731192068903/Put/seqid=0 2024-11-09T22:41:10,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742144_1320 (size=9657) 2024-11-09T22:41:10,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/6dc14b97c6144487977d2778efefb66e 2024-11-09T22:41:10,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/5ddaafaf455d425abab7eed8e5cacf3d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/5ddaafaf455d425abab7eed8e5cacf3d 2024-11-09T22:41:10,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/5ddaafaf455d425abab7eed8e5cacf3d, entries=300, sequenceid=17, filesize=55.4 K 2024-11-09T22:41:10,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/49f1ae14f02d411caf1cc28cfa46c065 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/49f1ae14f02d411caf1cc28cfa46c065 2024-11-09T22:41:10,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/49f1ae14f02d411caf1cc28cfa46c065, entries=100, sequenceid=17, filesize=9.4 K 2024-11-09T22:41:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/6dc14b97c6144487977d2778efefb66e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/6dc14b97c6144487977d2778efefb66e 2024-11-09T22:41:10,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/6dc14b97c6144487977d2778efefb66e, entries=100, sequenceid=17, filesize=9.4 K 2024-11-09T22:41:10,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1936e48909d1b634ac268f8f357af8e0 in 1315ms, sequenceid=17, compaction requested=false 2024-11-09T22:41:10,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:10,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T22:41:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:10,270 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T22:41:10,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cb24a918477144e2b81398b9d02ed0ae_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192068970/Put/seqid=0 2024-11-09T22:41:10,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742145_1321 (size=12154) 2024-11-09T22:41:10,491 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T22:41:10,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:10,735 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cb24a918477144e2b81398b9d02ed0ae_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cb24a918477144e2b81398b9d02ed0ae_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/23dda682297b4bb9b5f64c1c412f4eb9, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:10,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/23dda682297b4bb9b5f64c1c412f4eb9 is 175, key is test_row_0/A:col10/1731192068970/Put/seqid=0 2024-11-09T22:41:10,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742146_1322 (size=30955) 2024-11-09T22:41:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T22:41:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:11,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:11,140 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/23dda682297b4bb9b5f64c1c412f4eb9 2024-11-09T22:41:11,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192131141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/eb03443b8570419180ce1e622b3b615e is 50, key is test_row_0/B:col10/1731192068970/Put/seqid=0 2024-11-09T22:41:11,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742147_1323 (size=12001) 2024-11-09T22:41:11,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192131142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192131143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192131143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192131144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192131255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192131255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192131255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192131255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192131264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192131465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192131465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192131465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192131465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192131475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,552 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/eb03443b8570419180ce1e622b3b615e 2024-11-09T22:41:11,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/b902f4cfeb1941b2b2dd071ea6fe7aa3 is 50, key is test_row_0/C:col10/1731192068970/Put/seqid=0 2024-11-09T22:41:11,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742148_1324 (size=12001) 2024-11-09T22:41:11,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192131775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192131775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192131775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192131775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192131794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:11,962 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/b902f4cfeb1941b2b2dd071ea6fe7aa3 2024-11-09T22:41:11,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/23dda682297b4bb9b5f64c1c412f4eb9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/23dda682297b4bb9b5f64c1c412f4eb9 2024-11-09T22:41:11,969 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/23dda682297b4bb9b5f64c1c412f4eb9, entries=150, sequenceid=40, filesize=30.2 K 2024-11-09T22:41:11,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/eb03443b8570419180ce1e622b3b615e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/eb03443b8570419180ce1e622b3b615e 2024-11-09T22:41:11,974 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/eb03443b8570419180ce1e622b3b615e, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T22:41:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/b902f4cfeb1941b2b2dd071ea6fe7aa3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/b902f4cfeb1941b2b2dd071ea6fe7aa3 2024-11-09T22:41:11,978 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/b902f4cfeb1941b2b2dd071ea6fe7aa3, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T22:41:11,978 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1936e48909d1b634ac268f8f357af8e0 in 1708ms, sequenceid=40, compaction requested=false 2024-11-09T22:41:11,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:11,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:11,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-09T22:41:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-09T22:41:11,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-09T22:41:11,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0880 sec 2024-11-09T22:41:11,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.0950 sec 2024-11-09T22:41:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:12,289 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:12,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T22:41:12,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b81ec6183ceb44d1a2fa43add6271908_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192072288/Put/seqid=0 2024-11-09T22:41:12,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742149_1325 (size=12154) 2024-11-09T22:41:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192132342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192132342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192132343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192132343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192132353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192132455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192132465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192132465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192132465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192132465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192132665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192132675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192132675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192132684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192132684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,701 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:12,705 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b81ec6183ceb44d1a2fa43add6271908_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b81ec6183ceb44d1a2fa43add6271908_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:12,705 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/9ad8be44ea6d411487df28335027b8a3, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:12,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/9ad8be44ea6d411487df28335027b8a3 is 175, key is test_row_0/A:col10/1731192072288/Put/seqid=0 2024-11-09T22:41:12,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742150_1326 (size=30955) 2024-11-09T22:41:12,710 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/9ad8be44ea6d411487df28335027b8a3 2024-11-09T22:41:12,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/9f253610ad21402989f5c46e1939f899 is 50, key is test_row_0/B:col10/1731192072288/Put/seqid=0 2024-11-09T22:41:12,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742151_1327 (size=12001) 2024-11-09T22:41:12,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192132975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:12,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192132987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192132995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192133005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192133005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T22:41:13,045 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:13,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-09T22:41:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T22:41:13,047 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:13,047 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:13,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:13,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/9f253610ad21402989f5c46e1939f899 2024-11-09T22:41:13,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/402d1f6600164d718a02f26f913303bb is 50, key is test_row_0/C:col10/1731192072288/Put/seqid=0 2024-11-09T22:41:13,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742152_1328 (size=12001) 2024-11-09T22:41:13,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/402d1f6600164d718a02f26f913303bb 2024-11-09T22:41:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/9ad8be44ea6d411487df28335027b8a3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/9ad8be44ea6d411487df28335027b8a3 2024-11-09T22:41:13,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/9ad8be44ea6d411487df28335027b8a3, entries=150, sequenceid=55, filesize=30.2 K 2024-11-09T22:41:13,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/9f253610ad21402989f5c46e1939f899 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9f253610ad21402989f5c46e1939f899 2024-11-09T22:41:13,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9f253610ad21402989f5c46e1939f899, entries=150, sequenceid=55, filesize=11.7 K 2024-11-09T22:41:13,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/402d1f6600164d718a02f26f913303bb as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/402d1f6600164d718a02f26f913303bb 2024-11-09T22:41:13,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/402d1f6600164d718a02f26f913303bb, entries=150, sequenceid=55, filesize=11.7 K 2024-11-09T22:41:13,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1936e48909d1b634ac268f8f357af8e0 in 858ms, sequenceid=55, compaction requested=true 2024-11-09T22:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:13,147 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:13,148 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:13,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:13,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:13,148 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:13,148 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:13,148 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/A is initiating minor compaction (all files) 2024-11-09T22:41:13,148 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/B is initiating minor compaction (all files) 2024-11-09T22:41:13,148 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/A in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:13,148 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/B in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:13,149 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/5ddaafaf455d425abab7eed8e5cacf3d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/23dda682297b4bb9b5f64c1c412f4eb9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/9ad8be44ea6d411487df28335027b8a3] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=115.9 K 2024-11-09T22:41:13,149 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/49f1ae14f02d411caf1cc28cfa46c065, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/eb03443b8570419180ce1e622b3b615e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9f253610ad21402989f5c46e1939f899] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=32.9 K 2024-11-09T22:41:13,149 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:13,149 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/5ddaafaf455d425abab7eed8e5cacf3d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/23dda682297b4bb9b5f64c1c412f4eb9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/9ad8be44ea6d411487df28335027b8a3] 2024-11-09T22:41:13,149 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f1ae14f02d411caf1cc28cfa46c065, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731192068903 2024-11-09T22:41:13,149 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ddaafaf455d425abab7eed8e5cacf3d, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731192068897 2024-11-09T22:41:13,149 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting eb03443b8570419180ce1e622b3b615e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731192068959 2024-11-09T22:41:13,149 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23dda682297b4bb9b5f64c1c412f4eb9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731192068959 2024-11-09T22:41:13,149 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f253610ad21402989f5c46e1939f899, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731192071141 2024-11-09T22:41:13,149 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad8be44ea6d411487df28335027b8a3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731192071141 2024-11-09T22:41:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T22:41:13,156 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:13,158 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#B#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:13,158 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109b43c02e079444e8890ea58e6bf813a87_1936e48909d1b634ac268f8f357af8e0 store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:13,158 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/f890564303b74573be30ec443c17f2a3 is 50, key is test_row_0/B:col10/1731192072288/Put/seqid=0 2024-11-09T22:41:13,160 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109b43c02e079444e8890ea58e6bf813a87_1936e48909d1b634ac268f8f357af8e0, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:13,160 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b43c02e079444e8890ea58e6bf813a87_1936e48909d1b634ac268f8f357af8e0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:13,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742154_1330 (size=4469) 2024-11-09T22:41:13,174 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#A#compaction#282 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:13,175 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/b474440823a9409b9cd0959d7f8a1b1d is 175, key is test_row_0/A:col10/1731192072288/Put/seqid=0 2024-11-09T22:41:13,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742153_1329 (size=12104) 2024-11-09T22:41:13,187 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/f890564303b74573be30ec443c17f2a3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/f890564303b74573be30ec443c17f2a3 2024-11-09T22:41:13,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742155_1331 (size=31058) 2024-11-09T22:41:13,195 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/B of 1936e48909d1b634ac268f8f357af8e0 into f890564303b74573be30ec443c17f2a3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:13,195 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:13,195 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/B, priority=13, startTime=1731192073147; duration=0sec 2024-11-09T22:41:13,195 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:13,195 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:B 2024-11-09T22:41:13,195 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:13,195 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:13,196 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/C is initiating minor compaction (all files) 2024-11-09T22:41:13,196 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/C in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:13,196 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/6dc14b97c6144487977d2778efefb66e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/b902f4cfeb1941b2b2dd071ea6fe7aa3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/402d1f6600164d718a02f26f913303bb] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=32.9 K 2024-11-09T22:41:13,196 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc14b97c6144487977d2778efefb66e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731192068903 2024-11-09T22:41:13,196 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting b902f4cfeb1941b2b2dd071ea6fe7aa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731192068959 2024-11-09T22:41:13,196 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 402d1f6600164d718a02f26f913303bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731192071141 2024-11-09T22:41:13,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T22:41:13,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:13,201 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T22:41:13,205 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#C#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:13,206 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/08cf464ac0df4a14b98e635e28ac52da is 50, key is test_row_0/C:col10/1731192072288/Put/seqid=0 2024-11-09T22:41:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a4b8f064047a4b40978e0dcda6577bdc_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192072341/Put/seqid=0 2024-11-09T22:41:13,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742156_1332 (size=12104) 2024-11-09T22:41:13,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742157_1333 (size=12154) 2024-11-09T22:41:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T22:41:13,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192133525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192133525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192133525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192133527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192133533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,593 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/b474440823a9409b9cd0959d7f8a1b1d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b474440823a9409b9cd0959d7f8a1b1d 2024-11-09T22:41:13,597 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/A of 1936e48909d1b634ac268f8f357af8e0 into b474440823a9409b9cd0959d7f8a1b1d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:13,597 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:13,597 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/A, priority=13, startTime=1731192073147; duration=0sec 2024-11-09T22:41:13,598 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:13,598 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:A 2024-11-09T22:41:13,629 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/08cf464ac0df4a14b98e635e28ac52da as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/08cf464ac0df4a14b98e635e28ac52da 2024-11-09T22:41:13,634 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/C of 1936e48909d1b634ac268f8f357af8e0 into 08cf464ac0df4a14b98e635e28ac52da(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:13,634 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:13,634 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/C, priority=13, startTime=1731192073148; duration=0sec 2024-11-09T22:41:13,634 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:13,634 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:C 2024-11-09T22:41:13,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:13,641 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a4b8f064047a4b40978e0dcda6577bdc_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a4b8f064047a4b40978e0dcda6577bdc_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:13,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/1fbe511d6a7a4182ae441fd4d09fb9ba, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:13,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/1fbe511d6a7a4182ae441fd4d09fb9ba is 175, key is test_row_0/A:col10/1731192072341/Put/seqid=0 2024-11-09T22:41:13,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742158_1334 (size=30955) 2024-11-09T22:41:13,646 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/1fbe511d6a7a4182ae441fd4d09fb9ba 2024-11-09T22:41:13,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192133645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192133645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/4e27440f2d8147838432ae8f05e3edfa is 50, key is test_row_0/B:col10/1731192072341/Put/seqid=0 2024-11-09T22:41:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192133645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742159_1335 (size=12001) 2024-11-09T22:41:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T22:41:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192133855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192133865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192133865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,069 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/4e27440f2d8147838432ae8f05e3edfa 2024-11-09T22:41:14,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/5c806da41245419487723fbcabca9b15 is 50, key is test_row_0/C:col10/1731192072341/Put/seqid=0 2024-11-09T22:41:14,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742160_1336 (size=12001) 2024-11-09T22:41:14,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192134164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192134175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192134175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T22:41:14,479 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/5c806da41245419487723fbcabca9b15 2024-11-09T22:41:14,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/1fbe511d6a7a4182ae441fd4d09fb9ba as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/1fbe511d6a7a4182ae441fd4d09fb9ba 2024-11-09T22:41:14,487 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/1fbe511d6a7a4182ae441fd4d09fb9ba, entries=150, sequenceid=77, filesize=30.2 K 2024-11-09T22:41:14,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/4e27440f2d8147838432ae8f05e3edfa as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/4e27440f2d8147838432ae8f05e3edfa 2024-11-09T22:41:14,491 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/4e27440f2d8147838432ae8f05e3edfa, entries=150, sequenceid=77, filesize=11.7 K 2024-11-09T22:41:14,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/5c806da41245419487723fbcabca9b15 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/5c806da41245419487723fbcabca9b15 2024-11-09T22:41:14,495 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/5c806da41245419487723fbcabca9b15, entries=150, sequenceid=77, filesize=11.7 K 2024-11-09T22:41:14,496 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 1936e48909d1b634ac268f8f357af8e0 in 1296ms, sequenceid=77, compaction requested=false 2024-11-09T22:41:14,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:14,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:14,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-09T22:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-09T22:41:14,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-09T22:41:14,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4500 sec 2024-11-09T22:41:14,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4540 sec 2024-11-09T22:41:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:14,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T22:41:14,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d2fb934c97254517aab0b2852c57bcc1_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192073531/Put/seqid=0 2024-11-09T22:41:14,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742161_1337 (size=17034) 2024-11-09T22:41:14,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192134614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192134618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192134685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192134685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192134685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192134724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192134735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192134935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192134944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:14,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:14,968 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d2fb934c97254517aab0b2852c57bcc1_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d2fb934c97254517aab0b2852c57bcc1_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:14,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/753aeedc5ada41f8b04ba97d0964d22e, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:14,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/753aeedc5ada41f8b04ba97d0964d22e is 175, key is test_row_0/A:col10/1731192073531/Put/seqid=0 2024-11-09T22:41:14,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742162_1338 (size=48139) 2024-11-09T22:41:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T22:41:15,195 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:15,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-09T22:41:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T22:41:15,197 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:15,198 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:15,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192135245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:15,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192135255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T22:41:15,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T22:41:15,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:15,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,372 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/753aeedc5ada41f8b04ba97d0964d22e 2024-11-09T22:41:15,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/0e1d1fb84cc241a98b1d8ff36bfecc34 is 50, key is test_row_0/B:col10/1731192073531/Put/seqid=0 2024-11-09T22:41:15,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742163_1339 (size=12001) 2024-11-09T22:41:15,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T22:41:15,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T22:41:15,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T22:41:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192135704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:15,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192135704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:15,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192135704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:15,710 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:15,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:15,710 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:15,710 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_2' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192135765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192135765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:15,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/0e1d1fb84cc241a98b1d8ff36bfecc34 2024-11-09T22:41:15,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T22:41:15,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:15,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T22:41:15,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2dde4b7806aa4d0b94cb8ccfc83b0a01 is 50, key is test_row_0/C:col10/1731192073531/Put/seqid=0 2024-11-09T22:41:15,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742164_1340 (size=12001) 2024-11-09T22:41:15,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T22:41:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:15,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:16,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T22:41:16,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:16,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:16,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:16,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:16,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:16,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2dde4b7806aa4d0b94cb8ccfc83b0a01 2024-11-09T22:41:16,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/753aeedc5ada41f8b04ba97d0964d22e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/753aeedc5ada41f8b04ba97d0964d22e 2024-11-09T22:41:16,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/753aeedc5ada41f8b04ba97d0964d22e, entries=250, sequenceid=96, filesize=47.0 K 2024-11-09T22:41:16,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/0e1d1fb84cc241a98b1d8ff36bfecc34 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/0e1d1fb84cc241a98b1d8ff36bfecc34 2024-11-09T22:41:16,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/0e1d1fb84cc241a98b1d8ff36bfecc34, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T22:41:16,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2dde4b7806aa4d0b94cb8ccfc83b0a01 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2dde4b7806aa4d0b94cb8ccfc83b0a01 2024-11-09T22:41:16,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2dde4b7806aa4d0b94cb8ccfc83b0a01, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T22:41:16,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 1936e48909d1b634ac268f8f357af8e0 in 1712ms, sequenceid=96, compaction requested=true 2024-11-09T22:41:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:16,261 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:16,261 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:16,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:16,262 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:16,262 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/B is initiating minor compaction (all files) 2024-11-09T22:41:16,262 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/B in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:16,262 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:16,262 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/f890564303b74573be30ec443c17f2a3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/4e27440f2d8147838432ae8f05e3edfa, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/0e1d1fb84cc241a98b1d8ff36bfecc34] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=35.3 K 2024-11-09T22:41:16,262 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/A is initiating minor compaction (all files) 2024-11-09T22:41:16,262 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/A in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:16,262 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b474440823a9409b9cd0959d7f8a1b1d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/1fbe511d6a7a4182ae441fd4d09fb9ba, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/753aeedc5ada41f8b04ba97d0964d22e] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=107.6 K 2024-11-09T22:41:16,262 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:16,262 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b474440823a9409b9cd0959d7f8a1b1d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/1fbe511d6a7a4182ae441fd4d09fb9ba, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/753aeedc5ada41f8b04ba97d0964d22e] 2024-11-09T22:41:16,262 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting f890564303b74573be30ec443c17f2a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731192071141 2024-11-09T22:41:16,263 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b474440823a9409b9cd0959d7f8a1b1d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731192071141 2024-11-09T22:41:16,263 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e27440f2d8147838432ae8f05e3edfa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731192072341 2024-11-09T22:41:16,263 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fbe511d6a7a4182ae441fd4d09fb9ba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731192072341 2024-11-09T22:41:16,263 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1d1fb84cc241a98b1d8ff36bfecc34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731192073531 2024-11-09T22:41:16,263 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 753aeedc5ada41f8b04ba97d0964d22e, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731192073523 2024-11-09T22:41:16,270 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:16,274 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:16,274 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/e4361d2eb7404b8a88ca4d8ffa8f19f2 is 50, key is test_row_0/B:col10/1731192073531/Put/seqid=0 2024-11-09T22:41:16,275 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411099ac4a4aa522647af904b7270011b3325_1936e48909d1b634ac268f8f357af8e0 store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:16,277 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411099ac4a4aa522647af904b7270011b3325_1936e48909d1b634ac268f8f357af8e0, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:16,278 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099ac4a4aa522647af904b7270011b3325_1936e48909d1b634ac268f8f357af8e0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:16,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742165_1341 (size=12207) 2024-11-09T22:41:16,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T22:41:16,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742166_1342 (size=4469) 2024-11-09T22:41:16,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:16,289 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T22:41:16,297 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/e4361d2eb7404b8a88ca4d8ffa8f19f2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/e4361d2eb7404b8a88ca4d8ffa8f19f2 2024-11-09T22:41:16,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fa58f8d2f5224d6eaf8be357bb7f5327_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192074613/Put/seqid=0 2024-11-09T22:41:16,302 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/B of 1936e48909d1b634ac268f8f357af8e0 into e4361d2eb7404b8a88ca4d8ffa8f19f2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:16,302 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:16,302 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/B, priority=13, startTime=1731192076261; duration=0sec 2024-11-09T22:41:16,302 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:16,302 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:B 2024-11-09T22:41:16,302 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:16,303 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:16,303 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/C is initiating minor compaction (all files) 2024-11-09T22:41:16,303 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/C in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:16,303 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/08cf464ac0df4a14b98e635e28ac52da, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/5c806da41245419487723fbcabca9b15, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2dde4b7806aa4d0b94cb8ccfc83b0a01] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=35.3 K 2024-11-09T22:41:16,304 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 08cf464ac0df4a14b98e635e28ac52da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731192071141 2024-11-09T22:41:16,304 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c806da41245419487723fbcabca9b15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731192072341 2024-11-09T22:41:16,305 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dde4b7806aa4d0b94cb8ccfc83b0a01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731192073531 2024-11-09T22:41:16,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742167_1343 (size=12154) 2024-11-09T22:41:16,324 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:16,325 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/67956359afe54d95883bfa17199abfaf is 50, key is test_row_0/C:col10/1731192073531/Put/seqid=0 2024-11-09T22:41:16,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742168_1344 (size=12207) 2024-11-09T22:41:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T22:41:16,690 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#A#compaction#290 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:16,690 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/826656ed62f64b908ce13d0b310f003f is 175, key is test_row_0/A:col10/1731192073531/Put/seqid=0 2024-11-09T22:41:16,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742169_1345 (size=31161) 2024-11-09T22:41:16,697 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/826656ed62f64b908ce13d0b310f003f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/826656ed62f64b908ce13d0b310f003f 2024-11-09T22:41:16,702 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/A of 1936e48909d1b634ac268f8f357af8e0 into 826656ed62f64b908ce13d0b310f003f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:16,702 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:16,702 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/A, priority=13, startTime=1731192076261; duration=0sec 2024-11-09T22:41:16,702 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:16,702 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:A 2024-11-09T22:41:16,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:16,710 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fa58f8d2f5224d6eaf8be357bb7f5327_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fa58f8d2f5224d6eaf8be357bb7f5327_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/f802d3afcc5446c28765ee580e69eac5, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:16,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/f802d3afcc5446c28765ee580e69eac5 is 175, key is test_row_0/A:col10/1731192074613/Put/seqid=0 2024-11-09T22:41:16,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742170_1346 (size=30955) 2024-11-09T22:41:16,735 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/67956359afe54d95883bfa17199abfaf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/67956359afe54d95883bfa17199abfaf 2024-11-09T22:41:16,739 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/C of 1936e48909d1b634ac268f8f357af8e0 into 67956359afe54d95883bfa17199abfaf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:16,739 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:16,739 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/C, priority=13, startTime=1731192076261; duration=0sec 2024-11-09T22:41:16,739 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:16,739 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:C 2024-11-09T22:41:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:16,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192136824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:16,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192136826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:16,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192136935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:16,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192136935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:16,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:17,115 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/f802d3afcc5446c28765ee580e69eac5 2024-11-09T22:41:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/7b9e376c458b47d79288d75e2a4af82e is 50, key is test_row_0/B:col10/1731192074613/Put/seqid=0 2024-11-09T22:41:17,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742171_1347 (size=12001) 2024-11-09T22:41:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192137145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192137145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T22:41:17,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192137455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:17,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192137455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:17,525 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/7b9e376c458b47d79288d75e2a4af82e 2024-11-09T22:41:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/47d7028edd3b4fc0bc513150346ad264 is 50, key is test_row_0/C:col10/1731192074613/Put/seqid=0 2024-11-09T22:41:17,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742172_1348 (size=12001) 2024-11-09T22:41:17,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192137724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192137724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:17,732 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,732 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:17,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192137734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:17,741 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_2' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:17,946 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/47d7028edd3b4fc0bc513150346ad264 2024-11-09T22:41:17,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/f802d3afcc5446c28765ee580e69eac5 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/f802d3afcc5446c28765ee580e69eac5 2024-11-09T22:41:17,953 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/f802d3afcc5446c28765ee580e69eac5, entries=150, sequenceid=115, filesize=30.2 K 2024-11-09T22:41:17,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/7b9e376c458b47d79288d75e2a4af82e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/7b9e376c458b47d79288d75e2a4af82e 2024-11-09T22:41:17,957 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/7b9e376c458b47d79288d75e2a4af82e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-09T22:41:17,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/47d7028edd3b4fc0bc513150346ad264 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/47d7028edd3b4fc0bc513150346ad264 2024-11-09T22:41:17,961 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/47d7028edd3b4fc0bc513150346ad264, entries=150, sequenceid=115, filesize=11.7 K 2024-11-09T22:41:17,962 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 1936e48909d1b634ac268f8f357af8e0 in 1673ms, sequenceid=115, compaction requested=false 2024-11-09T22:41:17,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:17,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:17,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-09T22:41:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-09T22:41:17,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-09T22:41:17,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7660 sec 2024-11-09T22:41:17,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.7700 sec 2024-11-09T22:41:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:17,969 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:17,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T22:41:17,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110971054bf58bbc4f7387c71181628f4bd5_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192076825/Put/seqid=0 2024-11-09T22:41:17,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742173_1349 (size=14794) 2024-11-09T22:41:18,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192138015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:18,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192138016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:18,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192138125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192138135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:18,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192138335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:18,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192138345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:18,380 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:18,383 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110971054bf58bbc4f7387c71181628f4bd5_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110971054bf58bbc4f7387c71181628f4bd5_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:18,384 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/37969fa14f1b4e78bd629683e5656542, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:18,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/37969fa14f1b4e78bd629683e5656542 is 175, key is test_row_0/A:col10/1731192076825/Put/seqid=0 2024-11-09T22:41:18,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742174_1350 (size=39749) 2024-11-09T22:41:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192138645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:18,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192138655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:18,795 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/37969fa14f1b4e78bd629683e5656542 2024-11-09T22:41:18,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/1eeed423bd8f45598325a5567b69c73d is 50, key is test_row_0/B:col10/1731192076825/Put/seqid=0 2024-11-09T22:41:18,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742175_1351 (size=12151) 2024-11-09T22:41:19,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192139165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:19,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192139165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:19,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:19,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:19,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:19,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:19,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:19,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/1eeed423bd8f45598325a5567b69c73d 2024-11-09T22:41:19,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/7c38e87fe34244dfa684f3b07ba6ff1f is 50, key is test_row_0/C:col10/1731192076825/Put/seqid=0 2024-11-09T22:41:19,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742176_1352 (size=12151) 2024-11-09T22:41:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T22:41:19,365 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:19,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-09T22:41:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T22:41:19,367 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:19,367 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:19,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T22:41:19,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-09T22:41:19,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:19,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:19,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:19,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:19,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:19,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/7c38e87fe34244dfa684f3b07ba6ff1f 2024-11-09T22:41:19,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/37969fa14f1b4e78bd629683e5656542 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/37969fa14f1b4e78bd629683e5656542 2024-11-09T22:41:19,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/37969fa14f1b4e78bd629683e5656542, entries=200, sequenceid=136, filesize=38.8 K 2024-11-09T22:41:19,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/1eeed423bd8f45598325a5567b69c73d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/1eeed423bd8f45598325a5567b69c73d 2024-11-09T22:41:19,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/1eeed423bd8f45598325a5567b69c73d, entries=150, sequenceid=136, filesize=11.9 K 2024-11-09T22:41:19,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/7c38e87fe34244dfa684f3b07ba6ff1f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/7c38e87fe34244dfa684f3b07ba6ff1f 2024-11-09T22:41:19,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/7c38e87fe34244dfa684f3b07ba6ff1f, entries=150, sequenceid=136, filesize=11.9 K 2024-11-09T22:41:19,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 1936e48909d1b634ac268f8f357af8e0 in 1664ms, sequenceid=136, compaction requested=true 2024-11-09T22:41:19,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:19,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:19,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:19,634 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:19,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:19,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:19,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:19,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T22:41:19,634 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:19,635 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:19,635 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/B is initiating minor compaction (all files) 2024-11-09T22:41:19,635 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/B in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:19,635 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/e4361d2eb7404b8a88ca4d8ffa8f19f2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/7b9e376c458b47d79288d75e2a4af82e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/1eeed423bd8f45598325a5567b69c73d] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=35.5 K 2024-11-09T22:41:19,636 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:19,636 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/A is initiating minor compaction (all files) 2024-11-09T22:41:19,636 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/A in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:19,636 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/826656ed62f64b908ce13d0b310f003f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/f802d3afcc5446c28765ee580e69eac5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/37969fa14f1b4e78bd629683e5656542] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=99.5 K 2024-11-09T22:41:19,636 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:19,636 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/826656ed62f64b908ce13d0b310f003f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/f802d3afcc5446c28765ee580e69eac5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/37969fa14f1b4e78bd629683e5656542] 2024-11-09T22:41:19,636 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting e4361d2eb7404b8a88ca4d8ffa8f19f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731192073531 2024-11-09T22:41:19,637 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826656ed62f64b908ce13d0b310f003f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731192073531 2024-11-09T22:41:19,637 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b9e376c458b47d79288d75e2a4af82e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731192074612 2024-11-09T22:41:19,637 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f802d3afcc5446c28765ee580e69eac5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731192074612 2024-11-09T22:41:19,637 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eeed423bd8f45598325a5567b69c73d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731192076816 2024-11-09T22:41:19,638 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37969fa14f1b4e78bd629683e5656542, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731192076808 2024-11-09T22:41:19,650 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#B#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:19,650 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/555af7bc9aae400aa4b9bf3d0bf3b5e5 is 50, key is test_row_0/B:col10/1731192076825/Put/seqid=0 2024-11-09T22:41:19,659 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:19,664 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411093795ef2a46e84dcabae32b45224d1838_1936e48909d1b634ac268f8f357af8e0 store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:19,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742177_1353 (size=12459) 2024-11-09T22:41:19,667 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411093795ef2a46e84dcabae32b45224d1838_1936e48909d1b634ac268f8f357af8e0, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:19,667 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093795ef2a46e84dcabae32b45224d1838_1936e48909d1b634ac268f8f357af8e0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:19,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-09T22:41:19,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:19,673 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T22:41:19,673 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/555af7bc9aae400aa4b9bf3d0bf3b5e5 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/555af7bc9aae400aa4b9bf3d0bf3b5e5 2024-11-09T22:41:19,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742178_1354 (size=4469) 2024-11-09T22:41:19,679 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/B of 1936e48909d1b634ac268f8f357af8e0 into 555af7bc9aae400aa4b9bf3d0bf3b5e5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:19,679 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:19,679 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/B, priority=13, startTime=1731192079634; duration=0sec 2024-11-09T22:41:19,679 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:19,679 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:B 2024-11-09T22:41:19,679 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:19,680 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:19,680 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/C is initiating minor compaction (all files) 2024-11-09T22:41:19,680 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/C in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:19,680 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/67956359afe54d95883bfa17199abfaf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/47d7028edd3b4fc0bc513150346ad264, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/7c38e87fe34244dfa684f3b07ba6ff1f] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=35.5 K 2024-11-09T22:41:19,680 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 67956359afe54d95883bfa17199abfaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731192073531 2024-11-09T22:41:19,681 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d7028edd3b4fc0bc513150346ad264, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731192074612 2024-11-09T22:41:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a045ba2fbb014e7ea28be412134c6e71_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192078009/Put/seqid=0 2024-11-09T22:41:19,681 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c38e87fe34244dfa684f3b07ba6ff1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731192076816 2024-11-09T22:41:19,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742179_1355 (size=12304) 2024-11-09T22:41:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T22:41:19,688 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:19,688 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/18a90d1e17cd47e89836c75f0ff26d24 is 50, key is test_row_0/C:col10/1731192076825/Put/seqid=0 2024-11-09T22:41:19,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742180_1356 (size=12459) 2024-11-09T22:41:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T22:41:20,078 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#A#compaction#300 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:20,079 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/222b2044d6744f258b26e583b614c29a is 175, key is test_row_0/A:col10/1731192076825/Put/seqid=0 2024-11-09T22:41:20,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742181_1357 (size=31413) 2024-11-09T22:41:20,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:20,088 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a045ba2fbb014e7ea28be412134c6e71_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a045ba2fbb014e7ea28be412134c6e71_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:20,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/7acd904f199046e2a806024029b611e7, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:20,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/7acd904f199046e2a806024029b611e7 is 175, key is test_row_0/A:col10/1731192078009/Put/seqid=0 2024-11-09T22:41:20,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742182_1358 (size=31105) 2024-11-09T22:41:20,097 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/18a90d1e17cd47e89836c75f0ff26d24 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/18a90d1e17cd47e89836c75f0ff26d24 2024-11-09T22:41:20,101 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/C of 1936e48909d1b634ac268f8f357af8e0 into 18a90d1e17cd47e89836c75f0ff26d24(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:20,101 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:20,101 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/C, priority=13, startTime=1731192079634; duration=0sec 2024-11-09T22:41:20,101 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:20,101 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:C 2024-11-09T22:41:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:20,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:20,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192140224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:20,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192140228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:20,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:20,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192140335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:20,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192140335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:20,486 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/222b2044d6744f258b26e583b614c29a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/222b2044d6744f258b26e583b614c29a 2024-11-09T22:41:20,490 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/A of 1936e48909d1b634ac268f8f357af8e0 into 222b2044d6744f258b26e583b614c29a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:20,490 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:20,490 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/A, priority=13, startTime=1731192079633; duration=0sec 2024-11-09T22:41:20,490 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:20,490 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:A 2024-11-09T22:41:20,493 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/7acd904f199046e2a806024029b611e7 2024-11-09T22:41:20,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/ade2534424914af3be210ec37b049260 is 50, key is test_row_0/B:col10/1731192078009/Put/seqid=0 2024-11-09T22:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T22:41:20,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742183_1359 (size=12151) 2024-11-09T22:41:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192140555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192140555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192140865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:20,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192140865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:20,908 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/ade2534424914af3be210ec37b049260 2024-11-09T22:41:20,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2f1dfd6d5a2546338e606ddcce8eeccf is 50, key is test_row_0/C:col10/1731192078009/Put/seqid=0 2024-11-09T22:41:20,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742184_1360 (size=12151) 2024-11-09T22:41:20,918 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2f1dfd6d5a2546338e606ddcce8eeccf 2024-11-09T22:41:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/7acd904f199046e2a806024029b611e7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/7acd904f199046e2a806024029b611e7 2024-11-09T22:41:20,925 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/7acd904f199046e2a806024029b611e7, entries=150, sequenceid=154, filesize=30.4 K 2024-11-09T22:41:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/ade2534424914af3be210ec37b049260 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/ade2534424914af3be210ec37b049260 2024-11-09T22:41:20,930 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/ade2534424914af3be210ec37b049260, entries=150, sequenceid=154, filesize=11.9 K 2024-11-09T22:41:20,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2f1dfd6d5a2546338e606ddcce8eeccf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2f1dfd6d5a2546338e606ddcce8eeccf 2024-11-09T22:41:20,935 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2f1dfd6d5a2546338e606ddcce8eeccf, entries=150, sequenceid=154, filesize=11.9 K 2024-11-09T22:41:20,935 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 1936e48909d1b634ac268f8f357af8e0 in 1263ms, sequenceid=154, compaction requested=false 2024-11-09T22:41:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:20,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:20,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-09T22:41:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-09T22:41:20,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-09T22:41:20,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-09T22:41:20,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5750 sec 2024-11-09T22:41:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:21,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:21,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T22:41:21,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109162abb1fd8db440c939e38588df4a774_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192081378/Put/seqid=0 2024-11-09T22:41:21,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742185_1361 (size=17284) 2024-11-09T22:41:21,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192141423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192141428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T22:41:21,515 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:21,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-09T22:41:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T22:41:21,517 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:21,518 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:21,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:21,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192141544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192141544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T22:41:21,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T22:41:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:21,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:21,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192141755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47818 deadline: 1731192141765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192141765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47762 deadline: 1731192141765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,768 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,769 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_2' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47808 deadline: 1731192141775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:21,779 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 1936e48909d1b634ac268f8f357af8e0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:21,793 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:21,796 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109162abb1fd8db440c939e38588df4a774_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109162abb1fd8db440c939e38588df4a774_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:21,797 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/86199b4beff640b9aaba32678cedfce0, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:21,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/86199b4beff640b9aaba32678cedfce0 is 175, key is test_row_0/A:col10/1731192081378/Put/seqid=0 2024-11-09T22:41:21,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742186_1362 (size=48389) 2024-11-09T22:41:21,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/86199b4beff640b9aaba32678cedfce0 2024-11-09T22:41:21,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/64467ccb8a394d1aaef2aa79b3c75b63 is 50, key is test_row_0/B:col10/1731192081378/Put/seqid=0 2024-11-09T22:41:21,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742187_1363 (size=12151) 2024-11-09T22:41:21,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T22:41:21,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:21,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T22:41:21,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T22:41:21,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:21,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:21,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:21,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:21,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192142065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:22,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192142074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:22,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T22:41:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T22:41:22,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/64467ccb8a394d1aaef2aa79b3c75b63 2024-11-09T22:41:22,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/9e1c890b72ca430088d82e2bdcc6ad62 is 50, key is test_row_0/C:col10/1731192081378/Put/seqid=0 2024-11-09T22:41:22,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742188_1364 (size=12151) 2024-11-09T22:41:22,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T22:41:22,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:22,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T22:41:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T22:41:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:22,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192142585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192142585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:22,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/9e1c890b72ca430088d82e2bdcc6ad62 2024-11-09T22:41:22,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/86199b4beff640b9aaba32678cedfce0 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/86199b4beff640b9aaba32678cedfce0 2024-11-09T22:41:22,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/86199b4beff640b9aaba32678cedfce0, entries=250, sequenceid=177, filesize=47.3 K 2024-11-09T22:41:22,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/64467ccb8a394d1aaef2aa79b3c75b63 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/64467ccb8a394d1aaef2aa79b3c75b63 2024-11-09T22:41:22,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/64467ccb8a394d1aaef2aa79b3c75b63, entries=150, sequenceid=177, filesize=11.9 K 2024-11-09T22:41:22,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/9e1c890b72ca430088d82e2bdcc6ad62 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/9e1c890b72ca430088d82e2bdcc6ad62 2024-11-09T22:41:22,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/9e1c890b72ca430088d82e2bdcc6ad62, entries=150, sequenceid=177, filesize=11.9 K 2024-11-09T22:41:22,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1936e48909d1b634ac268f8f357af8e0 in 1261ms, sequenceid=177, compaction requested=true 2024-11-09T22:41:22,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:22,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:22,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:22,640 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:22,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:22,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:22,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:22,640 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:22,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:22,641 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:22,641 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:22,641 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/A is initiating minor compaction (all files) 2024-11-09T22:41:22,641 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/B is initiating minor compaction (all files) 2024-11-09T22:41:22,641 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/A in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,641 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/B in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,641 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/555af7bc9aae400aa4b9bf3d0bf3b5e5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/ade2534424914af3be210ec37b049260, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/64467ccb8a394d1aaef2aa79b3c75b63] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=35.9 K 2024-11-09T22:41:22,641 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/222b2044d6744f258b26e583b614c29a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/7acd904f199046e2a806024029b611e7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/86199b4beff640b9aaba32678cedfce0] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=108.3 K 2024-11-09T22:41:22,641 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,641 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/222b2044d6744f258b26e583b614c29a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/7acd904f199046e2a806024029b611e7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/86199b4beff640b9aaba32678cedfce0] 2024-11-09T22:41:22,642 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 555af7bc9aae400aa4b9bf3d0bf3b5e5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731192076816 2024-11-09T22:41:22,642 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 222b2044d6744f258b26e583b614c29a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731192076816 2024-11-09T22:41:22,642 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ade2534424914af3be210ec37b049260, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731192078009 2024-11-09T22:41:22,642 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7acd904f199046e2a806024029b611e7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731192078009 2024-11-09T22:41:22,643 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 64467ccb8a394d1aaef2aa79b3c75b63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731192080217 2024-11-09T22:41:22,643 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86199b4beff640b9aaba32678cedfce0, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731192080217 2024-11-09T22:41:22,648 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#B#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:22,649 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/b003b63aee7d4658ae0eac45e19e71f2 is 50, key is test_row_0/B:col10/1731192081378/Put/seqid=0 2024-11-09T22:41:22,649 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:22,654 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110983e5c2a9a2b748fc9ffa09a493e45494_1936e48909d1b634ac268f8f357af8e0 store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:22,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742189_1365 (size=12561) 2024-11-09T22:41:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T22:41:22,656 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110983e5c2a9a2b748fc9ffa09a493e45494_1936e48909d1b634ac268f8f357af8e0, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:22,656 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110983e5c2a9a2b748fc9ffa09a493e45494_1936e48909d1b634ac268f8f357af8e0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:22,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742190_1366 (size=4469) 2024-11-09T22:41:22,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T22:41:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:22,739 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T22:41:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e842ca47cc9243c6a14131681b6fbd39_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192081415/Put/seqid=0 2024-11-09T22:41:22,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742191_1367 (size=12304) 2024-11-09T22:41:23,060 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/b003b63aee7d4658ae0eac45e19e71f2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b003b63aee7d4658ae0eac45e19e71f2 2024-11-09T22:41:23,062 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:23,062 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/fc94490768b34befabe5a076c321b28b is 175, key is test_row_0/A:col10/1731192081378/Put/seqid=0 2024-11-09T22:41:23,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742192_1368 (size=31515) 2024-11-09T22:41:23,065 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/B of 1936e48909d1b634ac268f8f357af8e0 into b003b63aee7d4658ae0eac45e19e71f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:23,065 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:23,065 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/B, priority=13, startTime=1731192082640; duration=0sec 2024-11-09T22:41:23,065 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:23,066 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:B 2024-11-09T22:41:23,066 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:23,066 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:23,066 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/C is initiating minor compaction (all files) 2024-11-09T22:41:23,066 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/C in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:23,067 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/18a90d1e17cd47e89836c75f0ff26d24, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2f1dfd6d5a2546338e606ddcce8eeccf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/9e1c890b72ca430088d82e2bdcc6ad62] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=35.9 K 2024-11-09T22:41:23,067 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a90d1e17cd47e89836c75f0ff26d24, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731192076816 2024-11-09T22:41:23,067 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f1dfd6d5a2546338e606ddcce8eeccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731192078009 2024-11-09T22:41:23,067 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e1c890b72ca430088d82e2bdcc6ad62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731192080217 2024-11-09T22:41:23,072 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/fc94490768b34befabe5a076c321b28b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/fc94490768b34befabe5a076c321b28b 2024-11-09T22:41:23,073 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:23,074 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/4a5b3af353dd4c768987eb60b75408c4 is 50, key is test_row_0/C:col10/1731192081378/Put/seqid=0 2024-11-09T22:41:23,076 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/A of 1936e48909d1b634ac268f8f357af8e0 into fc94490768b34befabe5a076c321b28b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:23,076 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:23,076 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/A, priority=13, startTime=1731192082640; duration=0sec 2024-11-09T22:41:23,076 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:23,076 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:A 2024-11-09T22:41:23,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742193_1369 (size=12561) 2024-11-09T22:41:23,090 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/4a5b3af353dd4c768987eb60b75408c4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4a5b3af353dd4c768987eb60b75408c4 2024-11-09T22:41:23,094 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/C of 1936e48909d1b634ac268f8f357af8e0 into 4a5b3af353dd4c768987eb60b75408c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:23,095 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:23,095 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/C, priority=13, startTime=1731192082640; duration=0sec 2024-11-09T22:41:23,095 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:23,095 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:C 2024-11-09T22:41:23,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:23,152 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e842ca47cc9243c6a14131681b6fbd39_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e842ca47cc9243c6a14131681b6fbd39_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:23,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/a67716e55226439f9b8ff979fccd6c4b, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/a67716e55226439f9b8ff979fccd6c4b is 175, key is test_row_0/A:col10/1731192081415/Put/seqid=0 2024-11-09T22:41:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742194_1370 (size=31105) 2024-11-09T22:41:23,557 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/a67716e55226439f9b8ff979fccd6c4b 2024-11-09T22:41:23,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/8d3c13bb9b3f43d7b978327609b48ec8 is 50, key is test_row_0/B:col10/1731192081415/Put/seqid=0 2024-11-09T22:41:23,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742195_1371 (size=12151) 2024-11-09T22:41:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:23,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:23,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192143643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:23,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192143644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T22:41:23,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192143755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192143764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:23,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192143965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:23,968 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/8d3c13bb9b3f43d7b978327609b48ec8 2024-11-09T22:41:23,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/4fb21f7f6eb04d42855eb566b5176e90 is 50, key is test_row_0/C:col10/1731192081415/Put/seqid=0 2024-11-09T22:41:23,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192143975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:23,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742196_1372 (size=12151) 2024-11-09T22:41:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192144275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:24,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192144284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:24,379 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/4fb21f7f6eb04d42855eb566b5176e90 2024-11-09T22:41:24,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/a67716e55226439f9b8ff979fccd6c4b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a67716e55226439f9b8ff979fccd6c4b 2024-11-09T22:41:24,386 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a67716e55226439f9b8ff979fccd6c4b, entries=150, sequenceid=193, filesize=30.4 K 2024-11-09T22:41:24,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/8d3c13bb9b3f43d7b978327609b48ec8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/8d3c13bb9b3f43d7b978327609b48ec8 2024-11-09T22:41:24,390 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/8d3c13bb9b3f43d7b978327609b48ec8, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T22:41:24,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/4fb21f7f6eb04d42855eb566b5176e90 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4fb21f7f6eb04d42855eb566b5176e90 2024-11-09T22:41:24,395 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4fb21f7f6eb04d42855eb566b5176e90, entries=150, sequenceid=193, filesize=11.9 K 2024-11-09T22:41:24,396 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1936e48909d1b634ac268f8f357af8e0 in 1656ms, sequenceid=193, compaction requested=false 2024-11-09T22:41:24,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:24,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:24,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-09T22:41:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-09T22:41:24,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-09T22:41:24,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8790 sec 2024-11-09T22:41:24,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.8830 sec 2024-11-09T22:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:24,797 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:24,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T22:41:24,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f192889a010b43b9ae0a125918560990_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192084796/Put/seqid=0 2024-11-09T22:41:24,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742197_1373 (size=17284) 2024-11-09T22:41:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192144831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192144832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:24,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192144944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:24,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192144944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:25,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192145155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:25,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192145155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:25,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:25,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:25,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f192889a010b43b9ae0a125918560990_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f192889a010b43b9ae0a125918560990_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:25,215 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/c6fddc9e269644bfb4fd4de9f3731826, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:25,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/c6fddc9e269644bfb4fd4de9f3731826 is 175, key is test_row_0/A:col10/1731192084796/Put/seqid=0 2024-11-09T22:41:25,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742198_1374 (size=48389) 2024-11-09T22:41:25,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192145464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192145464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:25,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/c6fddc9e269644bfb4fd4de9f3731826 2024-11-09T22:41:25,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/18c9b973062c46dfbcdeb22a904b7b56 is 50, key is test_row_0/B:col10/1731192084796/Put/seqid=0 2024-11-09T22:41:25,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742199_1375 (size=12151) 2024-11-09T22:41:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T22:41:25,686 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:25,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-09T22:41:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T22:41:25,687 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:25,688 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:25,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T22:41:25,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T22:41:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:25,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:25,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192145975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:25,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192145975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:25,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T22:41:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:25,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T22:41:26,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/18c9b973062c46dfbcdeb22a904b7b56 2024-11-09T22:41:26,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/84a1bf5fe2a04a0a84f8504d6e62901e is 50, key is test_row_0/C:col10/1731192084796/Put/seqid=0 2024-11-09T22:41:26,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742200_1376 (size=12151) 2024-11-09T22:41:26,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T22:41:26,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:26,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T22:41:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T22:41:26,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T22:41:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:26,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/84a1bf5fe2a04a0a84f8504d6e62901e 2024-11-09T22:41:26,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/c6fddc9e269644bfb4fd4de9f3731826 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/c6fddc9e269644bfb4fd4de9f3731826 2024-11-09T22:41:26,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/c6fddc9e269644bfb4fd4de9f3731826, entries=250, sequenceid=218, filesize=47.3 K 2024-11-09T22:41:26,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/18c9b973062c46dfbcdeb22a904b7b56 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/18c9b973062c46dfbcdeb22a904b7b56 2024-11-09T22:41:26,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/18c9b973062c46dfbcdeb22a904b7b56, entries=150, sequenceid=218, filesize=11.9 K 2024-11-09T22:41:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/84a1bf5fe2a04a0a84f8504d6e62901e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/84a1bf5fe2a04a0a84f8504d6e62901e 2024-11-09T22:41:26,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/84a1bf5fe2a04a0a84f8504d6e62901e, entries=150, sequenceid=218, filesize=11.9 K 2024-11-09T22:41:26,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1936e48909d1b634ac268f8f357af8e0 in 1690ms, sequenceid=218, compaction requested=true 2024-11-09T22:41:26,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:26,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:26,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:26,487 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:26,487 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:26,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:26,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:26,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:26,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:26,488 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:26,488 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:26,488 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/B is initiating minor compaction (all files) 2024-11-09T22:41:26,488 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/A is initiating minor compaction (all files) 2024-11-09T22:41:26,488 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/B in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,488 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/A in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,488 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/fc94490768b34befabe5a076c321b28b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a67716e55226439f9b8ff979fccd6c4b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/c6fddc9e269644bfb4fd4de9f3731826] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=108.4 K 2024-11-09T22:41:26,488 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b003b63aee7d4658ae0eac45e19e71f2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/8d3c13bb9b3f43d7b978327609b48ec8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/18c9b973062c46dfbcdeb22a904b7b56] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=36.0 K 2024-11-09T22:41:26,488 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,488 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/fc94490768b34befabe5a076c321b28b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a67716e55226439f9b8ff979fccd6c4b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/c6fddc9e269644bfb4fd4de9f3731826] 2024-11-09T22:41:26,489 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting b003b63aee7d4658ae0eac45e19e71f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731192080217 2024-11-09T22:41:26,489 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc94490768b34befabe5a076c321b28b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731192080217 2024-11-09T22:41:26,489 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3c13bb9b3f43d7b978327609b48ec8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731192081389 2024-11-09T22:41:26,489 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a67716e55226439f9b8ff979fccd6c4b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731192081389 2024-11-09T22:41:26,489 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c9b973062c46dfbcdeb22a904b7b56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731192083643 2024-11-09T22:41:26,489 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6fddc9e269644bfb4fd4de9f3731826, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731192083632 2024-11-09T22:41:26,496 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:26,497 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/3b31c82a15354ed08fe485aa7a11335b is 50, key is test_row_0/B:col10/1731192084796/Put/seqid=0 2024-11-09T22:41:26,497 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:26,503 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109be869340f9be4e94adf0c1fb8744b5fb_1936e48909d1b634ac268f8f357af8e0 store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:26,506 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109be869340f9be4e94adf0c1fb8744b5fb_1936e48909d1b634ac268f8f357af8e0, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:26,506 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109be869340f9be4e94adf0c1fb8744b5fb_1936e48909d1b634ac268f8f357af8e0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:26,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742201_1377 (size=12663) 2024-11-09T22:41:26,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742202_1378 (size=4469) 2024-11-09T22:41:26,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T22:41:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,604 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T22:41:26,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109356187b19f8340d0b7d9789661abc56f_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192084830/Put/seqid=0 2024-11-09T22:41:26,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742203_1379 (size=12304) 2024-11-09T22:41:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T22:41:26,908 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/3b31c82a15354ed08fe485aa7a11335b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/3b31c82a15354ed08fe485aa7a11335b 2024-11-09T22:41:26,912 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/B of 1936e48909d1b634ac268f8f357af8e0 into 3b31c82a15354ed08fe485aa7a11335b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:26,912 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:26,912 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/B, priority=13, startTime=1731192086487; duration=0sec 2024-11-09T22:41:26,913 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:26,913 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:B 2024-11-09T22:41:26,913 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:26,913 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:26,913 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/C is initiating minor compaction (all files) 2024-11-09T22:41:26,913 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/C in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:26,914 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4a5b3af353dd4c768987eb60b75408c4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4fb21f7f6eb04d42855eb566b5176e90, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/84a1bf5fe2a04a0a84f8504d6e62901e] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=36.0 K 2024-11-09T22:41:26,914 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a5b3af353dd4c768987eb60b75408c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731192080217 2024-11-09T22:41:26,914 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb21f7f6eb04d42855eb566b5176e90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731192081389 2024-11-09T22:41:26,914 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 84a1bf5fe2a04a0a84f8504d6e62901e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731192083643 2024-11-09T22:41:26,915 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:26,915 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/191db4a2a06b41409041438e4e1b4f50 is 175, key is test_row_0/A:col10/1731192084796/Put/seqid=0 2024-11-09T22:41:26,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742204_1380 (size=31617) 2024-11-09T22:41:26,923 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:26,924 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2ef58de5b21f4877b782b8375ba2b75d is 50, key is test_row_0/C:col10/1731192084796/Put/seqid=0 2024-11-09T22:41:26,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742205_1381 (size=12663) 2024-11-09T22:41:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:26,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. as already flushing 2024-11-09T22:41:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:27,015 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109356187b19f8340d0b7d9789661abc56f_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109356187b19f8340d0b7d9789661abc56f_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:27,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/befe274f9a68403a97095f5bf21066c5, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:27,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/befe274f9a68403a97095f5bf21066c5 is 175, key is test_row_0/A:col10/1731192084830/Put/seqid=0 2024-11-09T22:41:27,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742206_1382 (size=31105) 2024-11-09T22:41:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192147048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192147049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:27,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192147165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:27,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192147165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:27,323 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/191db4a2a06b41409041438e4e1b4f50 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/191db4a2a06b41409041438e4e1b4f50 2024-11-09T22:41:27,327 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/A of 1936e48909d1b634ac268f8f357af8e0 into 191db4a2a06b41409041438e4e1b4f50(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:27,327 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:27,327 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/A, priority=13, startTime=1731192086487; duration=0sec 2024-11-09T22:41:27,327 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:27,327 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:A 2024-11-09T22:41:27,330 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2ef58de5b21f4877b782b8375ba2b75d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2ef58de5b21f4877b782b8375ba2b75d 2024-11-09T22:41:27,338 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/C of 1936e48909d1b634ac268f8f357af8e0 into 2ef58de5b21f4877b782b8375ba2b75d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:27,338 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:27,338 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/C, priority=13, startTime=1731192086487; duration=0sec 2024-11-09T22:41:27,338 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:27,338 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:C 2024-11-09T22:41:27,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192147375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192147375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:27,423 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/befe274f9a68403a97095f5bf21066c5 2024-11-09T22:41:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/2e336dc7939a44c18ff6d477833afc89 is 50, key is test_row_0/B:col10/1731192084830/Put/seqid=0 2024-11-09T22:41:27,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742207_1383 (size=12151) 2024-11-09T22:41:27,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192147685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192147685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:27,833 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/2e336dc7939a44c18ff6d477833afc89 2024-11-09T22:41:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T22:41:27,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/74800cf1b9eb44ef8ee3b7f521f76bf6 is 50, key is test_row_0/C:col10/1731192084830/Put/seqid=0 2024-11-09T22:41:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742208_1384 (size=12151) 2024-11-09T22:41:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47816 deadline: 1731192148195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:28,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731192148195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:41:28,248 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/74800cf1b9eb44ef8ee3b7f521f76bf6 2024-11-09T22:41:28,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/befe274f9a68403a97095f5bf21066c5 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/befe274f9a68403a97095f5bf21066c5 2024-11-09T22:41:28,255 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/befe274f9a68403a97095f5bf21066c5, entries=150, sequenceid=232, filesize=30.4 K 2024-11-09T22:41:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/2e336dc7939a44c18ff6d477833afc89 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/2e336dc7939a44c18ff6d477833afc89 2024-11-09T22:41:28,258 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/2e336dc7939a44c18ff6d477833afc89, entries=150, sequenceid=232, filesize=11.9 K 2024-11-09T22:41:28,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/74800cf1b9eb44ef8ee3b7f521f76bf6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/74800cf1b9eb44ef8ee3b7f521f76bf6 2024-11-09T22:41:28,262 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/74800cf1b9eb44ef8ee3b7f521f76bf6, entries=150, sequenceid=232, filesize=11.9 K 2024-11-09T22:41:28,263 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1936e48909d1b634ac268f8f357af8e0 in 1659ms, sequenceid=232, compaction requested=false 2024-11-09T22:41:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-09T22:41:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-09T22:41:28,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-09T22:41:28,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5760 sec 2024-11-09T22:41:28,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.5810 sec 2024-11-09T22:41:28,890 INFO [Thread-1459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1459. 2024-11-09T22:41:28,890 DEBUG [Thread-1459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:28,890 DEBUG [Thread-1459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,890 INFO [Thread-1461 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1461. 2024-11-09T22:41:28,890 DEBUG [Thread-1459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,890 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:28,891 DEBUG [Thread-1461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:28,891 DEBUG [Thread-1461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,891 DEBUG [Thread-1461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:28,895 INFO [Thread-1463 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1463. 2024-11-09T22:41:28,895 DEBUG [Thread-1463 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:28,895 DEBUG [Thread-1463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,895 DEBUG [Thread-1463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:28,897 INFO [Thread-1467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1467. 2024-11-09T22:41:28,897 DEBUG [Thread-1467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:28,897 DEBUG [Thread-1467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,897 DEBUG [Thread-1467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:28,897 INFO [Thread-1465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1465. 2024-11-09T22:41:28,897 DEBUG [Thread-1465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:28,897 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,897 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:28,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:29,208 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:29,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T22:41:29,209 INFO [Thread-1452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1452. 2024-11-09T22:41:29,209 DEBUG [Thread-1452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:29,209 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:29,209 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:29,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:29,217 INFO [Thread-1448 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1448. 2024-11-09T22:41:29,218 DEBUG [Thread-1448 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:29,218 DEBUG [Thread-1448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:29,218 DEBUG [Thread-1448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:29,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:29,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092797495933a94de69d99d521f3d096be_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192089206/Put/seqid=0 2024-11-09T22:41:29,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742209_1385 (size=12354) 2024-11-09T22:41:29,624 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:29,632 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092797495933a94de69d99d521f3d096be_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092797495933a94de69d99d521f3d096be_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:29,633 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/b9db2c2fceff4cc3ac7f02cc1a242f40, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:29,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/b9db2c2fceff4cc3ac7f02cc1a242f40 is 175, key is test_row_0/A:col10/1731192089206/Put/seqid=0 2024-11-09T22:41:29,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742210_1386 (size=31155) 2024-11-09T22:41:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T22:41:29,857 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:30,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/b9db2c2fceff4cc3ac7f02cc1a242f40 2024-11-09T22:41:30,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/9268d8ce373449d58bd0c547d09d0115 is 50, key is test_row_0/B:col10/1731192089206/Put/seqid=0 2024-11-09T22:41:30,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742211_1387 (size=12201) 2024-11-09T22:41:30,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/9268d8ce373449d58bd0c547d09d0115 2024-11-09T22:41:30,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/be05cd2897284c86b65f554e345e25fd is 50, key is test_row_0/C:col10/1731192089206/Put/seqid=0 2024-11-09T22:41:30,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742212_1388 (size=12201) 2024-11-09T22:41:30,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/be05cd2897284c86b65f554e345e25fd 2024-11-09T22:41:30,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/b9db2c2fceff4cc3ac7f02cc1a242f40 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b9db2c2fceff4cc3ac7f02cc1a242f40 2024-11-09T22:41:30,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b9db2c2fceff4cc3ac7f02cc1a242f40, entries=150, sequenceid=258, filesize=30.4 K 2024-11-09T22:41:30,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/9268d8ce373449d58bd0c547d09d0115 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9268d8ce373449d58bd0c547d09d0115 2024-11-09T22:41:30,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9268d8ce373449d58bd0c547d09d0115, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T22:41:30,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/be05cd2897284c86b65f554e345e25fd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/be05cd2897284c86b65f554e345e25fd 2024-11-09T22:41:30,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/be05cd2897284c86b65f554e345e25fd, entries=150, sequenceid=258, filesize=11.9 K 2024-11-09T22:41:30,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=6.71 KB/6870 for 1936e48909d1b634ac268f8f357af8e0 in 1696ms, sequenceid=258, compaction requested=true 2024-11-09T22:41:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:30,904 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1936e48909d1b634ac268f8f357af8e0:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:30,904 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:30,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:30,905 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:30,905 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:30,905 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/A is initiating minor compaction (all files) 2024-11-09T22:41:30,905 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/B is initiating minor compaction (all files) 2024-11-09T22:41:30,905 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/B in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:30,905 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/A in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:30,905 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/3b31c82a15354ed08fe485aa7a11335b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/2e336dc7939a44c18ff6d477833afc89, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9268d8ce373449d58bd0c547d09d0115] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=36.1 K 2024-11-09T22:41:30,905 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/191db4a2a06b41409041438e4e1b4f50, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/befe274f9a68403a97095f5bf21066c5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b9db2c2fceff4cc3ac7f02cc1a242f40] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=91.7 K 2024-11-09T22:41:30,905 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:30,905 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/191db4a2a06b41409041438e4e1b4f50, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/befe274f9a68403a97095f5bf21066c5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b9db2c2fceff4cc3ac7f02cc1a242f40] 2024-11-09T22:41:30,906 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b31c82a15354ed08fe485aa7a11335b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731192083643 2024-11-09T22:41:30,906 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 191db4a2a06b41409041438e4e1b4f50, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731192083643 2024-11-09T22:41:30,906 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e336dc7939a44c18ff6d477833afc89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731192084826 2024-11-09T22:41:30,906 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting befe274f9a68403a97095f5bf21066c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731192084826 2024-11-09T22:41:30,906 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9268d8ce373449d58bd0c547d09d0115, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731192087025 2024-11-09T22:41:30,906 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9db2c2fceff4cc3ac7f02cc1a242f40, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731192087025 2024-11-09T22:41:30,911 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:30,912 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:30,912 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110970a9ce73aafb451a862a64b264dab704_1936e48909d1b634ac268f8f357af8e0 store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:30,912 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/92a876abb1bc4249a7359a48f415b2e3 is 50, key is test_row_0/B:col10/1731192089206/Put/seqid=0 2024-11-09T22:41:30,915 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110970a9ce73aafb451a862a64b264dab704_1936e48909d1b634ac268f8f357af8e0, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:30,916 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110970a9ce73aafb451a862a64b264dab704_1936e48909d1b634ac268f8f357af8e0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:30,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742213_1389 (size=12815) 2024-11-09T22:41:30,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742214_1390 (size=4469) 2024-11-09T22:41:31,373 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#A#compaction#327 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:31,374 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/847e4e43571640aeb762784744110da1 is 175, key is test_row_0/A:col10/1731192089206/Put/seqid=0 2024-11-09T22:41:31,377 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/92a876abb1bc4249a7359a48f415b2e3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/92a876abb1bc4249a7359a48f415b2e3 2024-11-09T22:41:31,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742215_1391 (size=31769) 2024-11-09T22:41:31,383 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/B of 1936e48909d1b634ac268f8f357af8e0 into 92a876abb1bc4249a7359a48f415b2e3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:31,383 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:31,383 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/B, priority=13, startTime=1731192090904; duration=0sec 2024-11-09T22:41:31,383 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:31,383 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:B 2024-11-09T22:41:31,383 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:31,384 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:31,384 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 1936e48909d1b634ac268f8f357af8e0/C is initiating minor compaction (all files) 2024-11-09T22:41:31,384 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1936e48909d1b634ac268f8f357af8e0/C in TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:31,384 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2ef58de5b21f4877b782b8375ba2b75d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/74800cf1b9eb44ef8ee3b7f521f76bf6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/be05cd2897284c86b65f554e345e25fd] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp, totalSize=36.1 K 2024-11-09T22:41:31,385 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef58de5b21f4877b782b8375ba2b75d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731192083643 2024-11-09T22:41:31,385 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 74800cf1b9eb44ef8ee3b7f521f76bf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731192084826 2024-11-09T22:41:31,385 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting be05cd2897284c86b65f554e345e25fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731192087025 2024-11-09T22:41:31,393 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1936e48909d1b634ac268f8f357af8e0#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:31,394 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2a454ac55d054dc0979321ded69ef4f3 is 50, key is test_row_0/C:col10/1731192089206/Put/seqid=0 2024-11-09T22:41:31,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742216_1392 (size=12815) 2024-11-09T22:41:31,795 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/847e4e43571640aeb762784744110da1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/847e4e43571640aeb762784744110da1 2024-11-09T22:41:31,801 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/A of 1936e48909d1b634ac268f8f357af8e0 into 847e4e43571640aeb762784744110da1(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:31,801 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:31,801 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/A, priority=13, startTime=1731192090904; duration=0sec 2024-11-09T22:41:31,801 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:31,801 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:A 2024-11-09T22:41:31,801 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/2a454ac55d054dc0979321ded69ef4f3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2a454ac55d054dc0979321ded69ef4f3 2024-11-09T22:41:31,805 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1936e48909d1b634ac268f8f357af8e0/C of 1936e48909d1b634ac268f8f357af8e0 into 2a454ac55d054dc0979321ded69ef4f3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:31,805 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1936e48909d1b634ac268f8f357af8e0: 2024-11-09T22:41:31,805 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., storeName=1936e48909d1b634ac268f8f357af8e0/C, priority=13, startTime=1731192090904; duration=0sec 2024-11-09T22:41:31,805 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:31,805 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1936e48909d1b634ac268f8f357af8e0:C 2024-11-09T22:41:31,806 INFO [Thread-1456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1456. 2024-11-09T22:41:31,806 DEBUG [Thread-1456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:31,806 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:31,806 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:31,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:31,846 INFO [Thread-1450 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1450. 2024-11-09T22:41:31,846 DEBUG [Thread-1450 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:31,846 DEBUG [Thread-1450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:31,846 DEBUG [Thread-1450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:31,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:31,866 INFO [Thread-1454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1454. 2024-11-09T22:41:31,866 DEBUG [Thread-1454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:31,867 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:31,867 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:31,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1472 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4415 rows 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1477 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4428 rows 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1475 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4423 rows 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1474 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4414 rows 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1478 2024-11-09T22:41:31,867 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4430 rows 2024-11-09T22:41:31,868 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:41:31,868 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:41:31,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:31,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:31,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:31,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T22:41:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T22:41:31,874 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192091874"}]},"ts":"1731192091874"} 2024-11-09T22:41:31,876 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T22:41:31,876 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T22:41:31,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:41:31,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, UNASSIGN}] 2024-11-09T22:41:31,879 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, UNASSIGN 2024-11-09T22:41:31,880 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=1936e48909d1b634ac268f8f357af8e0, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:31,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, UNASSIGN because future has completed 2024-11-09T22:41:31,881 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:41:31,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:41:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T22:41:32,035 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:32,035 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:41:32,035 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 1936e48909d1b634ac268f8f357af8e0, disabling compactions & flushes 2024-11-09T22:41:32,035 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:32,035 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:32,035 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. after waiting 0 ms 2024-11-09T22:41:32,035 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:32,036 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 1936e48909d1b634ac268f8f357af8e0 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T22:41:32,048 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091af69b925d6047adaee662dd065007ae_1936e48909d1b634ac268f8f357af8e0 is 50, key is test_row_0/A:col10/1731192091865/Put/seqid=0 2024-11-09T22:41:32,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742217_1393 (size=9914) 2024-11-09T22:41:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T22:41:32,452 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:32,457 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091af69b925d6047adaee662dd065007ae_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091af69b925d6047adaee662dd065007ae_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:32,458 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/a176d9241c8446b7b4ae1640a4d6a8eb, store: [table=TestAcidGuarantees family=A region=1936e48909d1b634ac268f8f357af8e0] 2024-11-09T22:41:32,459 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/a176d9241c8446b7b4ae1640a4d6a8eb is 175, key is test_row_0/A:col10/1731192091865/Put/seqid=0 2024-11-09T22:41:32,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742218_1394 (size=22561) 2024-11-09T22:41:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T22:41:32,865 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/a176d9241c8446b7b4ae1640a4d6a8eb 2024-11-09T22:41:32,877 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/b52623c52229402397f1fba79f212aac is 50, key is test_row_0/B:col10/1731192091865/Put/seqid=0 2024-11-09T22:41:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742219_1395 (size=9857) 2024-11-09T22:41:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T22:41:33,282 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/b52623c52229402397f1fba79f212aac 2024-11-09T22:41:33,294 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/59cb483fb84a48e5886d3b6e508bdf5f is 50, key is test_row_0/C:col10/1731192091865/Put/seqid=0 2024-11-09T22:41:33,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742220_1396 (size=9857) 2024-11-09T22:41:33,698 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/59cb483fb84a48e5886d3b6e508bdf5f 2024-11-09T22:41:33,702 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/A/a176d9241c8446b7b4ae1640a4d6a8eb as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a176d9241c8446b7b4ae1640a4d6a8eb 2024-11-09T22:41:33,705 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a176d9241c8446b7b4ae1640a4d6a8eb, entries=100, sequenceid=268, filesize=22.0 K 2024-11-09T22:41:33,706 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/B/b52623c52229402397f1fba79f212aac as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b52623c52229402397f1fba79f212aac 2024-11-09T22:41:33,709 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b52623c52229402397f1fba79f212aac, entries=100, sequenceid=268, filesize=9.6 K 2024-11-09T22:41:33,710 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/.tmp/C/59cb483fb84a48e5886d3b6e508bdf5f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/59cb483fb84a48e5886d3b6e508bdf5f 2024-11-09T22:41:33,714 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/59cb483fb84a48e5886d3b6e508bdf5f, entries=100, sequenceid=268, filesize=9.6 K 2024-11-09T22:41:33,715 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1936e48909d1b634ac268f8f357af8e0 in 1680ms, sequenceid=268, compaction requested=false 2024-11-09T22:41:33,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/5ddaafaf455d425abab7eed8e5cacf3d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/23dda682297b4bb9b5f64c1c412f4eb9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b474440823a9409b9cd0959d7f8a1b1d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/9ad8be44ea6d411487df28335027b8a3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/1fbe511d6a7a4182ae441fd4d09fb9ba, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/753aeedc5ada41f8b04ba97d0964d22e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/826656ed62f64b908ce13d0b310f003f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/f802d3afcc5446c28765ee580e69eac5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/37969fa14f1b4e78bd629683e5656542, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/222b2044d6744f258b26e583b614c29a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/7acd904f199046e2a806024029b611e7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/86199b4beff640b9aaba32678cedfce0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/fc94490768b34befabe5a076c321b28b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a67716e55226439f9b8ff979fccd6c4b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/c6fddc9e269644bfb4fd4de9f3731826, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/191db4a2a06b41409041438e4e1b4f50, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/befe274f9a68403a97095f5bf21066c5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b9db2c2fceff4cc3ac7f02cc1a242f40] to archive 2024-11-09T22:41:33,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:33,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/5ddaafaf455d425abab7eed8e5cacf3d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/5ddaafaf455d425abab7eed8e5cacf3d 2024-11-09T22:41:33,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/23dda682297b4bb9b5f64c1c412f4eb9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/23dda682297b4bb9b5f64c1c412f4eb9 2024-11-09T22:41:33,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b474440823a9409b9cd0959d7f8a1b1d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b474440823a9409b9cd0959d7f8a1b1d 2024-11-09T22:41:33,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/9ad8be44ea6d411487df28335027b8a3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/9ad8be44ea6d411487df28335027b8a3 2024-11-09T22:41:33,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/1fbe511d6a7a4182ae441fd4d09fb9ba to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/1fbe511d6a7a4182ae441fd4d09fb9ba 2024-11-09T22:41:33,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/753aeedc5ada41f8b04ba97d0964d22e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/753aeedc5ada41f8b04ba97d0964d22e 2024-11-09T22:41:33,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/826656ed62f64b908ce13d0b310f003f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/826656ed62f64b908ce13d0b310f003f 2024-11-09T22:41:33,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/f802d3afcc5446c28765ee580e69eac5 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/f802d3afcc5446c28765ee580e69eac5 2024-11-09T22:41:33,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/37969fa14f1b4e78bd629683e5656542 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/37969fa14f1b4e78bd629683e5656542 2024-11-09T22:41:33,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/222b2044d6744f258b26e583b614c29a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/222b2044d6744f258b26e583b614c29a 2024-11-09T22:41:33,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/7acd904f199046e2a806024029b611e7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/7acd904f199046e2a806024029b611e7 2024-11-09T22:41:33,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/86199b4beff640b9aaba32678cedfce0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/86199b4beff640b9aaba32678cedfce0 2024-11-09T22:41:33,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/fc94490768b34befabe5a076c321b28b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/fc94490768b34befabe5a076c321b28b 2024-11-09T22:41:33,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a67716e55226439f9b8ff979fccd6c4b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a67716e55226439f9b8ff979fccd6c4b 2024-11-09T22:41:33,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/c6fddc9e269644bfb4fd4de9f3731826 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/c6fddc9e269644bfb4fd4de9f3731826 2024-11-09T22:41:33,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/191db4a2a06b41409041438e4e1b4f50 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/191db4a2a06b41409041438e4e1b4f50 2024-11-09T22:41:33,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/befe274f9a68403a97095f5bf21066c5 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/befe274f9a68403a97095f5bf21066c5 2024-11-09T22:41:33,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b9db2c2fceff4cc3ac7f02cc1a242f40 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/b9db2c2fceff4cc3ac7f02cc1a242f40 2024-11-09T22:41:33,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/49f1ae14f02d411caf1cc28cfa46c065, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/eb03443b8570419180ce1e622b3b615e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/f890564303b74573be30ec443c17f2a3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9f253610ad21402989f5c46e1939f899, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/4e27440f2d8147838432ae8f05e3edfa, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/e4361d2eb7404b8a88ca4d8ffa8f19f2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/0e1d1fb84cc241a98b1d8ff36bfecc34, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/7b9e376c458b47d79288d75e2a4af82e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/555af7bc9aae400aa4b9bf3d0bf3b5e5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/1eeed423bd8f45598325a5567b69c73d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/ade2534424914af3be210ec37b049260, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b003b63aee7d4658ae0eac45e19e71f2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/64467ccb8a394d1aaef2aa79b3c75b63, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/8d3c13bb9b3f43d7b978327609b48ec8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/3b31c82a15354ed08fe485aa7a11335b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/18c9b973062c46dfbcdeb22a904b7b56, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/2e336dc7939a44c18ff6d477833afc89, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9268d8ce373449d58bd0c547d09d0115] to archive 2024-11-09T22:41:33,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:33,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/49f1ae14f02d411caf1cc28cfa46c065 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/49f1ae14f02d411caf1cc28cfa46c065 2024-11-09T22:41:33,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/eb03443b8570419180ce1e622b3b615e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/eb03443b8570419180ce1e622b3b615e 2024-11-09T22:41:33,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/f890564303b74573be30ec443c17f2a3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/f890564303b74573be30ec443c17f2a3 2024-11-09T22:41:33,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9f253610ad21402989f5c46e1939f899 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9f253610ad21402989f5c46e1939f899 2024-11-09T22:41:33,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/4e27440f2d8147838432ae8f05e3edfa to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/4e27440f2d8147838432ae8f05e3edfa 2024-11-09T22:41:33,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/e4361d2eb7404b8a88ca4d8ffa8f19f2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/e4361d2eb7404b8a88ca4d8ffa8f19f2 2024-11-09T22:41:33,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/0e1d1fb84cc241a98b1d8ff36bfecc34 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/0e1d1fb84cc241a98b1d8ff36bfecc34 2024-11-09T22:41:33,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/7b9e376c458b47d79288d75e2a4af82e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/7b9e376c458b47d79288d75e2a4af82e 2024-11-09T22:41:33,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/555af7bc9aae400aa4b9bf3d0bf3b5e5 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/555af7bc9aae400aa4b9bf3d0bf3b5e5 2024-11-09T22:41:33,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/1eeed423bd8f45598325a5567b69c73d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/1eeed423bd8f45598325a5567b69c73d 2024-11-09T22:41:33,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/ade2534424914af3be210ec37b049260 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/ade2534424914af3be210ec37b049260 2024-11-09T22:41:33,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b003b63aee7d4658ae0eac45e19e71f2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b003b63aee7d4658ae0eac45e19e71f2 2024-11-09T22:41:33,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/64467ccb8a394d1aaef2aa79b3c75b63 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/64467ccb8a394d1aaef2aa79b3c75b63 2024-11-09T22:41:33,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/8d3c13bb9b3f43d7b978327609b48ec8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/8d3c13bb9b3f43d7b978327609b48ec8 2024-11-09T22:41:33,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/3b31c82a15354ed08fe485aa7a11335b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/3b31c82a15354ed08fe485aa7a11335b 2024-11-09T22:41:33,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/18c9b973062c46dfbcdeb22a904b7b56 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/18c9b973062c46dfbcdeb22a904b7b56 2024-11-09T22:41:33,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/2e336dc7939a44c18ff6d477833afc89 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/2e336dc7939a44c18ff6d477833afc89 2024-11-09T22:41:33,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9268d8ce373449d58bd0c547d09d0115 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/9268d8ce373449d58bd0c547d09d0115 2024-11-09T22:41:33,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/6dc14b97c6144487977d2778efefb66e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/b902f4cfeb1941b2b2dd071ea6fe7aa3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/08cf464ac0df4a14b98e635e28ac52da, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/402d1f6600164d718a02f26f913303bb, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/5c806da41245419487723fbcabca9b15, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/67956359afe54d95883bfa17199abfaf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2dde4b7806aa4d0b94cb8ccfc83b0a01, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/47d7028edd3b4fc0bc513150346ad264, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/18a90d1e17cd47e89836c75f0ff26d24, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/7c38e87fe34244dfa684f3b07ba6ff1f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2f1dfd6d5a2546338e606ddcce8eeccf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4a5b3af353dd4c768987eb60b75408c4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/9e1c890b72ca430088d82e2bdcc6ad62, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4fb21f7f6eb04d42855eb566b5176e90, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2ef58de5b21f4877b782b8375ba2b75d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/84a1bf5fe2a04a0a84f8504d6e62901e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/74800cf1b9eb44ef8ee3b7f521f76bf6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/be05cd2897284c86b65f554e345e25fd] to archive 2024-11-09T22:41:33,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:33,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/6dc14b97c6144487977d2778efefb66e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/6dc14b97c6144487977d2778efefb66e 2024-11-09T22:41:33,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/b902f4cfeb1941b2b2dd071ea6fe7aa3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/b902f4cfeb1941b2b2dd071ea6fe7aa3 2024-11-09T22:41:33,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/08cf464ac0df4a14b98e635e28ac52da to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/08cf464ac0df4a14b98e635e28ac52da 2024-11-09T22:41:33,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/402d1f6600164d718a02f26f913303bb to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/402d1f6600164d718a02f26f913303bb 2024-11-09T22:41:33,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/5c806da41245419487723fbcabca9b15 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/5c806da41245419487723fbcabca9b15 2024-11-09T22:41:33,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/67956359afe54d95883bfa17199abfaf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/67956359afe54d95883bfa17199abfaf 2024-11-09T22:41:33,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2dde4b7806aa4d0b94cb8ccfc83b0a01 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2dde4b7806aa4d0b94cb8ccfc83b0a01 2024-11-09T22:41:33,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/47d7028edd3b4fc0bc513150346ad264 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/47d7028edd3b4fc0bc513150346ad264 2024-11-09T22:41:33,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/18a90d1e17cd47e89836c75f0ff26d24 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/18a90d1e17cd47e89836c75f0ff26d24 2024-11-09T22:41:33,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/7c38e87fe34244dfa684f3b07ba6ff1f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/7c38e87fe34244dfa684f3b07ba6ff1f 2024-11-09T22:41:33,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2f1dfd6d5a2546338e606ddcce8eeccf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2f1dfd6d5a2546338e606ddcce8eeccf 2024-11-09T22:41:33,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4a5b3af353dd4c768987eb60b75408c4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4a5b3af353dd4c768987eb60b75408c4 2024-11-09T22:41:33,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/9e1c890b72ca430088d82e2bdcc6ad62 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/9e1c890b72ca430088d82e2bdcc6ad62 2024-11-09T22:41:33,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4fb21f7f6eb04d42855eb566b5176e90 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/4fb21f7f6eb04d42855eb566b5176e90 2024-11-09T22:41:33,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2ef58de5b21f4877b782b8375ba2b75d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2ef58de5b21f4877b782b8375ba2b75d 2024-11-09T22:41:33,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/84a1bf5fe2a04a0a84f8504d6e62901e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/84a1bf5fe2a04a0a84f8504d6e62901e 2024-11-09T22:41:33,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/74800cf1b9eb44ef8ee3b7f521f76bf6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/74800cf1b9eb44ef8ee3b7f521f76bf6 2024-11-09T22:41:33,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/be05cd2897284c86b65f554e345e25fd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/be05cd2897284c86b65f554e345e25fd 2024-11-09T22:41:33,769 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/recovered.edits/271.seqid, newMaxSeqId=271, maxSeqId=4 2024-11-09T22:41:33,769 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. 2024-11-09T22:41:33,770 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 1936e48909d1b634ac268f8f357af8e0: Waiting for close lock at 1731192092035Running coprocessor pre-close hooks at 1731192092035Disabling compacts and flushes for region at 1731192092035Disabling writes for close at 1731192092035Obtaining lock to block concurrent updates at 1731192092036 (+1 ms)Preparing flush snapshotting stores in 1936e48909d1b634ac268f8f357af8e0 at 1731192092036Finished memstore snapshotting TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731192092036Flushing stores of TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0. at 1731192092038 (+2 ms)Flushing 1936e48909d1b634ac268f8f357af8e0/A: creating writer at 1731192092038Flushing mob file 1936e48909d1b634ac268f8f357af8e0/A: appending metadata at 1731192092048 (+10 ms)Flushing mob file 1936e48909d1b634ac268f8f357af8e0/A: closing flushed file at 1731192092048Flushing 1936e48909d1b634ac268f8f357af8e0/A: appending metadata at 1731192092458 (+410 ms)Flushing 1936e48909d1b634ac268f8f357af8e0/A: closing flushed file at 1731192092458Flushing 1936e48909d1b634ac268f8f357af8e0/B: creating writer at 1731192092874 (+416 ms)Flushing 1936e48909d1b634ac268f8f357af8e0/B: appending metadata at 1731192092876 (+2 ms)Flushing 1936e48909d1b634ac268f8f357af8e0/B: closing flushed file at 1731192092876Flushing 1936e48909d1b634ac268f8f357af8e0/C: creating writer at 1731192093291 (+415 ms)Flushing 1936e48909d1b634ac268f8f357af8e0/C: appending metadata at 1731192093293 (+2 ms)Flushing 1936e48909d1b634ac268f8f357af8e0/C: closing flushed file at 1731192093293Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@515a5e63: reopening flushed file at 1731192093701 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cfbf5af: reopening flushed file at 1731192093705 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@107b1867: reopening flushed file at 1731192093709 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1936e48909d1b634ac268f8f357af8e0 in 1680ms, sequenceid=268, compaction requested=false at 1731192093715 (+6 ms)Writing region close event to WAL at 1731192093767 (+52 ms)Running coprocessor post-close hooks at 1731192093769 (+2 ms)Closed at 1731192093769 2024-11-09T22:41:33,771 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:33,772 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=1936e48909d1b634ac268f8f357af8e0, regionState=CLOSED 2024-11-09T22:41:33,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:41:33,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-09T22:41:33,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 1936e48909d1b634ac268f8f357af8e0, server=9d8c37854f6e,35101,1731191981027 in 1.8920 sec 2024-11-09T22:41:33,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-09T22:41:33,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1936e48909d1b634ac268f8f357af8e0, UNASSIGN in 1.8970 sec 2024-11-09T22:41:33,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-09T22:41:33,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9000 sec 2024-11-09T22:41:33,778 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192093778"}]},"ts":"1731192093778"} 2024-11-09T22:41:33,779 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T22:41:33,779 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T22:41:33,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9110 sec 2024-11-09T22:41:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T22:41:34,026 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:34,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T22:41:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:34,032 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T22:41:34,033 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:34,037 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,040 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/recovered.edits] 2024-11-09T22:41:34,043 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/847e4e43571640aeb762784744110da1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/847e4e43571640aeb762784744110da1 2024-11-09T22:41:34,045 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a176d9241c8446b7b4ae1640a4d6a8eb to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/A/a176d9241c8446b7b4ae1640a4d6a8eb 2024-11-09T22:41:34,048 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/92a876abb1bc4249a7359a48f415b2e3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/92a876abb1bc4249a7359a48f415b2e3 2024-11-09T22:41:34,050 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b52623c52229402397f1fba79f212aac to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/B/b52623c52229402397f1fba79f212aac 2024-11-09T22:41:34,052 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2a454ac55d054dc0979321ded69ef4f3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/2a454ac55d054dc0979321ded69ef4f3 2024-11-09T22:41:34,053 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/59cb483fb84a48e5886d3b6e508bdf5f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/C/59cb483fb84a48e5886d3b6e508bdf5f 2024-11-09T22:41:34,055 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/recovered.edits/271.seqid to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0/recovered.edits/271.seqid 2024-11-09T22:41:34,056 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,056 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T22:41:34,056 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T22:41:34,057 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T22:41:34,059 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109162abb1fd8db440c939e38588df4a774_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109162abb1fd8db440c939e38588df4a774_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,060 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091af69b925d6047adaee662dd065007ae_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091af69b925d6047adaee662dd065007ae_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,061 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092797495933a94de69d99d521f3d096be_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092797495933a94de69d99d521f3d096be_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,062 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109356187b19f8340d0b7d9789661abc56f_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109356187b19f8340d0b7d9789661abc56f_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,063 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110971054bf58bbc4f7387c71181628f4bd5_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110971054bf58bbc4f7387c71181628f4bd5_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,064 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a045ba2fbb014e7ea28be412134c6e71_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a045ba2fbb014e7ea28be412134c6e71_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,065 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a4b8f064047a4b40978e0dcda6577bdc_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a4b8f064047a4b40978e0dcda6577bdc_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,066 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b81ec6183ceb44d1a2fa43add6271908_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b81ec6183ceb44d1a2fa43add6271908_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,067 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cb24a918477144e2b81398b9d02ed0ae_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cb24a918477144e2b81398b9d02ed0ae_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,068 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d2fb934c97254517aab0b2852c57bcc1_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d2fb934c97254517aab0b2852c57bcc1_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,069 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e842ca47cc9243c6a14131681b6fbd39_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e842ca47cc9243c6a14131681b6fbd39_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,069 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f192889a010b43b9ae0a125918560990_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f192889a010b43b9ae0a125918560990_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,070 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fa58f8d2f5224d6eaf8be357bb7f5327_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fa58f8d2f5224d6eaf8be357bb7f5327_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,071 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb79c52b3189485b830b630af61bf563_1936e48909d1b634ac268f8f357af8e0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb79c52b3189485b830b630af61bf563_1936e48909d1b634ac268f8f357af8e0 2024-11-09T22:41:34,071 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T22:41:34,073 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:34,074 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T22:41:34,076 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T22:41:34,077 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:34,077 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T22:41:34,077 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731192094077"}]},"ts":"9223372036854775807"} 2024-11-09T22:41:34,079 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T22:41:34,079 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1936e48909d1b634ac268f8f357af8e0, NAME => 'TestAcidGuarantees,,1731192065746.1936e48909d1b634ac268f8f357af8e0.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T22:41:34,079 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T22:41:34,079 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731192094079"}]},"ts":"9223372036854775807"} 2024-11-09T22:41:34,081 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T22:41:34,081 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:34,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-09T22:41:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T22:41:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T22:41:34,140 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:34,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 240), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 566), ProcessCount=11 (was 11), AvailableMemoryMB=17637 (was 17676) 2024-11-09T22:41:34,162 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=17637 2024-11-09T22:41:34,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:41:34,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:41:34,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:34,167 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T22:41:34,167 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:34,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-09T22:41:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-09T22:41:34,168 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T22:41:34,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742221_1397 (size=959) 2024-11-09T22:41:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-09T22:41:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-09T22:41:34,579 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 2024-11-09T22:41:34,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742222_1398 (size=53) 2024-11-09T22:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-09T22:41:34,991 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:41:34,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bd15f4850dbe30ecfd33655486a40c35, disabling compactions & flushes 2024-11-09T22:41:34,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:34,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:34,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. after waiting 0 ms 2024-11-09T22:41:34,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:34,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:34,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd15f4850dbe30ecfd33655486a40c35: Waiting for close lock at 1731192094991Disabling compacts and flushes for region at 1731192094991Disabling writes for close at 1731192094992 (+1 ms)Writing region close event to WAL at 1731192094992Closed at 1731192094992 2024-11-09T22:41:34,995 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T22:41:34,996 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731192094995"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731192094995"}]},"ts":"1731192094995"} 2024-11-09T22:41:34,998 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T22:41:34,999 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T22:41:34,999 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192094999"}]},"ts":"1731192094999"} 2024-11-09T22:41:35,001 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T22:41:35,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, ASSIGN}] 2024-11-09T22:41:35,002 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, ASSIGN 2024-11-09T22:41:35,003 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, ASSIGN; state=OFFLINE, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=false 2024-11-09T22:41:35,155 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=bd15f4850dbe30ecfd33655486a40c35, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:35,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, ASSIGN because future has completed 2024-11-09T22:41:35,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:41:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-09T22:41:35,318 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:35,318 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:41:35,319 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,319 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:41:35,319 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,319 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,320 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,321 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd15f4850dbe30ecfd33655486a40c35 columnFamilyName A 2024-11-09T22:41:35,321 DEBUG [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:35,322 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] regionserver.HStore(327): Store=bd15f4850dbe30ecfd33655486a40c35/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:35,322 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,323 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd15f4850dbe30ecfd33655486a40c35 columnFamilyName B 2024-11-09T22:41:35,323 DEBUG [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:35,323 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] regionserver.HStore(327): Store=bd15f4850dbe30ecfd33655486a40c35/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:35,323 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,324 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd15f4850dbe30ecfd33655486a40c35 columnFamilyName C 2024-11-09T22:41:35,324 DEBUG [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:41:35,325 INFO [StoreOpener-bd15f4850dbe30ecfd33655486a40c35-1 {}] regionserver.HStore(327): Store=bd15f4850dbe30ecfd33655486a40c35/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:41:35,325 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,325 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,326 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,326 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,326 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,327 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:41:35,328 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,329 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:41:35,329 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened bd15f4850dbe30ecfd33655486a40c35; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71333076, jitterRate=0.0629456639289856}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:41:35,330 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:35,330 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for bd15f4850dbe30ecfd33655486a40c35: Running coprocessor pre-open hook at 1731192095319Writing region info on filesystem at 1731192095319Initializing all the Stores at 1731192095320 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192095320Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192095320Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192095320Cleaning up temporary data from old regions at 1731192095326 (+6 ms)Running coprocessor post-open hooks at 1731192095330 (+4 ms)Region opened successfully at 1731192095330 2024-11-09T22:41:35,331 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., pid=120, masterSystemTime=1731192095313 2024-11-09T22:41:35,332 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:35,332 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:35,333 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=bd15f4850dbe30ecfd33655486a40c35, regionState=OPEN, openSeqNum=2, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:35,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:41:35,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-09T22:41:35,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 in 175 msec 2024-11-09T22:41:35,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-09T22:41:35,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, ASSIGN in 335 msec 2024-11-09T22:41:35,338 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T22:41:35,338 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192095338"}]},"ts":"1731192095338"} 2024-11-09T22:41:35,339 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T22:41:35,339 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T22:41:35,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-09T22:41:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-09T22:41:36,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:36,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,318 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f57163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,321 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c0b326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,323 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,327 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T22:41:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d8c37854f6e,34651,1731191980160 2024-11-09T22:41:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d054315 2024-11-09T22:41:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T22:41:36,329 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T22:41:36,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,332 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fb9351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,334 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67283abe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,337 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@601a380a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,339 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@496bd032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,341 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@704d2da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,343 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef8ced0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,344 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5043673d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@235968da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,346 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573f0448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,348 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ed3599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,349 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acd3c5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,350 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae1a8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,352 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230dd7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,353 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6567b342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,355 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686146cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,356 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62246310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,357 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6a1164, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,359 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7717d676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:41:36,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:41:36,360 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:41:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:41:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:41:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fd47a8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:41:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:41:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:36,361 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:41:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e9b890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:41:36,362 DEBUG [Thread-1805 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,362 DEBUG [Thread-1822 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,362 DEBUG [Thread-1807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,362 DEBUG [Thread-1820 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,362 DEBUG [Thread-1814 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,362 DEBUG [Thread-1811 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,363 DEBUG [Thread-1803 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,363 DEBUG [Thread-1809 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,363 DEBUG [Thread-1816 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,364 DEBUG [Thread-1818 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:41:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,364 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,365 DEBUG [Thread-1818 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:41:36,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,365 DEBUG [Thread-1818 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:41:36,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-09T22:41:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T22:41:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,370 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2] 2024-11-09T22:41:36,371 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:36,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:36,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:36,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:41:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:36,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a8af89a87cc9425ebab946a2d8d343de is 50, key is test_row_0/A:col10/1731192096418/Put/seqid=0 2024-11-09T22:41:36,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192156444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742223_1399 (size=14341) 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192156448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192156449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192156453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192156455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T22:41:36,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:36,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:36,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:36,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192156565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192156565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192156565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192156565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192156565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:36,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T22:41:36,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192156775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192156775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192156775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192156775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192156775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:36,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:36,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a8af89a87cc9425ebab946a2d8d343de 2024-11-09T22:41:36,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/2c549bae678b4eea8eb224135665dd8f is 50, key is test_row_0/B:col10/1731192096418/Put/seqid=0 2024-11-09T22:41:36,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742224_1400 (size=12001) 2024-11-09T22:41:36,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:36,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T22:41:37,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192157085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192157085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192157085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192157085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192157085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/2c549bae678b4eea8eb224135665dd8f 2024-11-09T22:41:37,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:37,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:37,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/ff0aad08f26e47aea0fc220464c68f95 is 50, key is test_row_0/C:col10/1731192096418/Put/seqid=0 2024-11-09T22:41:37,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742225_1401 (size=12001) 2024-11-09T22:41:37,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T22:41:37,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:37,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:37,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:37,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192157595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192157595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192157595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192157605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192157605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:37,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/ff0aad08f26e47aea0fc220464c68f95 2024-11-09T22:41:37,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a8af89a87cc9425ebab946a2d8d343de as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8af89a87cc9425ebab946a2d8d343de 2024-11-09T22:41:37,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8af89a87cc9425ebab946a2d8d343de, entries=200, sequenceid=14, filesize=14.0 K 2024-11-09T22:41:37,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/2c549bae678b4eea8eb224135665dd8f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2c549bae678b4eea8eb224135665dd8f 2024-11-09T22:41:37,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2c549bae678b4eea8eb224135665dd8f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T22:41:37,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/ff0aad08f26e47aea0fc220464c68f95 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/ff0aad08f26e47aea0fc220464c68f95 2024-11-09T22:41:37,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/ff0aad08f26e47aea0fc220464c68f95, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T22:41:37,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for bd15f4850dbe30ecfd33655486a40c35 in 1304ms, sequenceid=14, compaction requested=false 2024-11-09T22:41:37,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:37,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-09T22:41:37,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:37,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T22:41:37,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/1b759710b1ad44e7847e18985c9eaf56 is 50, key is test_row_0/A:col10/1731192096437/Put/seqid=0 2024-11-09T22:41:37,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742226_1402 (size=12001) 2024-11-09T22:41:38,160 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/1b759710b1ad44e7847e18985c9eaf56 2024-11-09T22:41:38,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/d7b7cee47a23455d9dce7ded1dd2b8e4 is 50, key is test_row_0/B:col10/1731192096437/Put/seqid=0 2024-11-09T22:41:38,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742227_1403 (size=12001) 2024-11-09T22:41:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T22:41:38,570 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/d7b7cee47a23455d9dce7ded1dd2b8e4 2024-11-09T22:41:38,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/cc4e51c632b24a3583a7db3013b2cf02 is 50, key is test_row_0/C:col10/1731192096437/Put/seqid=0 2024-11-09T22:41:38,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742228_1404 (size=12001) 2024-11-09T22:41:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:38,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192158645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192158653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192158653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192158654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192158655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192158765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192158765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192158765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192158766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192158775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192158974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192158974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192158975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192158975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,988 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/cc4e51c632b24a3583a7db3013b2cf02 2024-11-09T22:41:38,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192158985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/1b759710b1ad44e7847e18985c9eaf56 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1b759710b1ad44e7847e18985c9eaf56 2024-11-09T22:41:38,993 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1b759710b1ad44e7847e18985c9eaf56, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T22:41:38,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/d7b7cee47a23455d9dce7ded1dd2b8e4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d7b7cee47a23455d9dce7ded1dd2b8e4 2024-11-09T22:41:38,997 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d7b7cee47a23455d9dce7ded1dd2b8e4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T22:41:38,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/cc4e51c632b24a3583a7db3013b2cf02 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/cc4e51c632b24a3583a7db3013b2cf02 2024-11-09T22:41:39,000 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/cc4e51c632b24a3583a7db3013b2cf02, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T22:41:39,000 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for bd15f4850dbe30ecfd33655486a40c35 in 1255ms, sequenceid=37, compaction requested=false 2024-11-09T22:41:39,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:39,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:39,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-09T22:41:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-09T22:41:39,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-09T22:41:39,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-11-09T22:41:39,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6390 sec 2024-11-09T22:41:39,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T22:41:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:39,289 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:39,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T22:41:39,294 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T22:41:39,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/445cb116b40d4187b44bfca04ceef26e is 50, key is test_row_0/A:col10/1731192099288/Put/seqid=0 2024-11-09T22:41:39,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742229_1405 (size=16681) 2024-11-09T22:41:39,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192159320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192159321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192159328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192159331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192159333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192159445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192159445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192159445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192159445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192159454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192159655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192159665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192159665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192159665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192159665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/445cb116b40d4187b44bfca04ceef26e 2024-11-09T22:41:39,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/821790a688d443e0a0e0f23c8ffe6a31 is 50, key is test_row_0/B:col10/1731192099288/Put/seqid=0 2024-11-09T22:41:39,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742230_1406 (size=12001) 2024-11-09T22:41:39,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192159975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192159985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192159985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192159985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192159985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/821790a688d443e0a0e0f23c8ffe6a31 2024-11-09T22:41:40,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd is 50, key is test_row_0/C:col10/1731192099288/Put/seqid=0 2024-11-09T22:41:40,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742231_1407 (size=12001) 2024-11-09T22:41:40,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd 2024-11-09T22:41:40,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/445cb116b40d4187b44bfca04ceef26e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/445cb116b40d4187b44bfca04ceef26e 2024-11-09T22:41:40,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/445cb116b40d4187b44bfca04ceef26e, entries=250, sequenceid=53, filesize=16.3 K 2024-11-09T22:41:40,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/821790a688d443e0a0e0f23c8ffe6a31 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/821790a688d443e0a0e0f23c8ffe6a31 2024-11-09T22:41:40,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/821790a688d443e0a0e0f23c8ffe6a31, entries=150, sequenceid=53, filesize=11.7 K 2024-11-09T22:41:40,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd 2024-11-09T22:41:40,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd, entries=150, sequenceid=53, filesize=11.7 K 2024-11-09T22:41:40,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for bd15f4850dbe30ecfd33655486a40c35 in 851ms, sequenceid=53, compaction requested=true 2024-11-09T22:41:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:40,141 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:40,141 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:40,142 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:40,142 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:40,142 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/B is initiating minor compaction (all files) 2024-11-09T22:41:40,142 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/A is initiating minor compaction (all files) 2024-11-09T22:41:40,142 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/B in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:40,142 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/A in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:40,142 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2c549bae678b4eea8eb224135665dd8f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d7b7cee47a23455d9dce7ded1dd2b8e4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/821790a688d443e0a0e0f23c8ffe6a31] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=35.2 K 2024-11-09T22:41:40,142 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8af89a87cc9425ebab946a2d8d343de, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1b759710b1ad44e7847e18985c9eaf56, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/445cb116b40d4187b44bfca04ceef26e] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=42.0 K 2024-11-09T22:41:40,142 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8af89a87cc9425ebab946a2d8d343de, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731192096415 2024-11-09T22:41:40,142 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c549bae678b4eea8eb224135665dd8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731192096415 2024-11-09T22:41:40,142 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b759710b1ad44e7847e18985c9eaf56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731192096437 2024-11-09T22:41:40,142 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b7cee47a23455d9dce7ded1dd2b8e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731192096437 2024-11-09T22:41:40,143 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 445cb116b40d4187b44bfca04ceef26e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731192098652 2024-11-09T22:41:40,143 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 821790a688d443e0a0e0f23c8ffe6a31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731192098652 2024-11-09T22:41:40,149 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#A#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:40,149 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/092c651b8f574023a0f8d8feb6b38f91 is 50, key is test_row_0/A:col10/1731192099288/Put/seqid=0 2024-11-09T22:41:40,151 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:40,152 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/27c1a2ee967b417596a9686c92801600 is 50, key is test_row_0/B:col10/1731192099288/Put/seqid=0 2024-11-09T22:41:40,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742232_1408 (size=12104) 2024-11-09T22:41:40,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742233_1409 (size=12104) 2024-11-09T22:41:40,171 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/27c1a2ee967b417596a9686c92801600 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/27c1a2ee967b417596a9686c92801600 2024-11-09T22:41:40,177 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/B of bd15f4850dbe30ecfd33655486a40c35 into 27c1a2ee967b417596a9686c92801600(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:40,177 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:40,177 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/B, priority=13, startTime=1731192100141; duration=0sec 2024-11-09T22:41:40,177 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:40,177 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:B 2024-11-09T22:41:40,177 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:40,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:40,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/C is initiating minor compaction (all files) 2024-11-09T22:41:40,178 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/C in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:40,178 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/ff0aad08f26e47aea0fc220464c68f95, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/cc4e51c632b24a3583a7db3013b2cf02, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=35.2 K 2024-11-09T22:41:40,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ff0aad08f26e47aea0fc220464c68f95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731192096415 2024-11-09T22:41:40,178 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting cc4e51c632b24a3583a7db3013b2cf02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731192096437 2024-11-09T22:41:40,179 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d5c98bf16d9b4f14a7fbf54bcae0ecfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731192098652 2024-11-09T22:41:40,185 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#C#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:40,186 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/34c16c0e63da4b8ba51f597bf17176fc is 50, key is test_row_0/C:col10/1731192099288/Put/seqid=0 2024-11-09T22:41:40,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742234_1410 (size=12104) 2024-11-09T22:41:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:40,489 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:40,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T22:41:40,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/34983a4bfdfb4d9daa61f81133861fa5 is 50, key is test_row_0/A:col10/1731192100488/Put/seqid=0 2024-11-09T22:41:40,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742235_1411 (size=14341) 2024-11-09T22:41:40,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192160511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192160512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192160513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192160514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192160515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-09T22:41:40,555 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:40,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-09T22:41:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T22:41:40,557 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:40,558 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:40,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:40,568 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/092c651b8f574023a0f8d8feb6b38f91 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/092c651b8f574023a0f8d8feb6b38f91 2024-11-09T22:41:40,571 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/A of bd15f4850dbe30ecfd33655486a40c35 into 092c651b8f574023a0f8d8feb6b38f91(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:40,571 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:40,571 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/A, priority=13, startTime=1731192100141; duration=0sec 2024-11-09T22:41:40,572 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:40,572 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:A 2024-11-09T22:41:40,593 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/34c16c0e63da4b8ba51f597bf17176fc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/34c16c0e63da4b8ba51f597bf17176fc 2024-11-09T22:41:40,596 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/C of bd15f4850dbe30ecfd33655486a40c35 into 34c16c0e63da4b8ba51f597bf17176fc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:40,597 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:40,597 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/C, priority=13, startTime=1731192100141; duration=0sec 2024-11-09T22:41:40,597 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:40,597 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:C 2024-11-09T22:41:40,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192160625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192160625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192160625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192160635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192160635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T22:41:40,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T22:41:40,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:40,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:40,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:40,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:40,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:40,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192160835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192160835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192160835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192160845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192160845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:40,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T22:41:40,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:40,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:40,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:40,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T22:41:40,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/34983a4bfdfb4d9daa61f81133861fa5 2024-11-09T22:41:40,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/86d2421bb2094667a025f7b773534699 is 50, key is test_row_0/B:col10/1731192100488/Put/seqid=0 2024-11-09T22:41:40,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742236_1412 (size=12001) 2024-11-09T22:41:41,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T22:41:41,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:41,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:41,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:41,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192161145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192161145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192161145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192161155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192161155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T22:41:41,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:41,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:41,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:41,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T22:41:41,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/86d2421bb2094667a025f7b773534699 2024-11-09T22:41:41,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/8b6b756f15504024871780b4e466d6bf is 50, key is test_row_0/C:col10/1731192100488/Put/seqid=0 2024-11-09T22:41:41,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742237_1413 (size=12001) 2024-11-09T22:41:41,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/8b6b756f15504024871780b4e466d6bf 2024-11-09T22:41:41,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T22:41:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:41,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:41,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/34983a4bfdfb4d9daa61f81133861fa5 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/34983a4bfdfb4d9daa61f81133861fa5 2024-11-09T22:41:41,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/34983a4bfdfb4d9daa61f81133861fa5, entries=200, sequenceid=75, filesize=14.0 K 2024-11-09T22:41:41,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/86d2421bb2094667a025f7b773534699 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/86d2421bb2094667a025f7b773534699 2024-11-09T22:41:41,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/86d2421bb2094667a025f7b773534699, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T22:41:41,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/8b6b756f15504024871780b4e466d6bf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/8b6b756f15504024871780b4e466d6bf 2024-11-09T22:41:41,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/8b6b756f15504024871780b4e466d6bf, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T22:41:41,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for bd15f4850dbe30ecfd33655486a40c35 in 851ms, sequenceid=75, compaction requested=false 2024-11-09T22:41:41,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:41,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-09T22:41:41,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:41,474 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T22:41:41,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/796490c1bcfc41a0bb21a21069bca0ce is 50, key is test_row_0/A:col10/1731192100513/Put/seqid=0 2024-11-09T22:41:41,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742238_1414 (size=12001) 2024-11-09T22:41:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T22:41:41,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192161687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192161688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192161697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192161698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192161699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192161805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192161805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192161805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192161805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192161815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:41,932 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/796490c1bcfc41a0bb21a21069bca0ce 2024-11-09T22:41:41,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/1f14afa785294c3bb12a0d09564285d2 is 50, key is test_row_0/B:col10/1731192100513/Put/seqid=0 2024-11-09T22:41:41,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742239_1415 (size=12001) 2024-11-09T22:41:42,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192162015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192162015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192162025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192162025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192162035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192162325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192162325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,342 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/1f14afa785294c3bb12a0d09564285d2 2024-11-09T22:41:42,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192162335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/a436dd45365840dfa99a186fc790bfaf is 50, key is test_row_0/C:col10/1731192100513/Put/seqid=0 2024-11-09T22:41:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192162345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192162345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742240_1416 (size=12001) 2024-11-09T22:41:42,354 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/a436dd45365840dfa99a186fc790bfaf 2024-11-09T22:41:42,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/796490c1bcfc41a0bb21a21069bca0ce as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/796490c1bcfc41a0bb21a21069bca0ce 2024-11-09T22:41:42,361 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/796490c1bcfc41a0bb21a21069bca0ce, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T22:41:42,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/1f14afa785294c3bb12a0d09564285d2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/1f14afa785294c3bb12a0d09564285d2 2024-11-09T22:41:42,366 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/1f14afa785294c3bb12a0d09564285d2, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T22:41:42,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/a436dd45365840dfa99a186fc790bfaf as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/a436dd45365840dfa99a186fc790bfaf 2024-11-09T22:41:42,371 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/a436dd45365840dfa99a186fc790bfaf, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T22:41:42,372 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for bd15f4850dbe30ecfd33655486a40c35 in 898ms, sequenceid=92, compaction requested=true 2024-11-09T22:41:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-09T22:41:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-09T22:41:42,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-09T22:41:42,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-09T22:41:42,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8190 sec 2024-11-09T22:41:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T22:41:42,705 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:42,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-09T22:41:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T22:41:42,707 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:42,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:42,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T22:41:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:42,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T22:41:42,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/5bc273063b7441f1af5da6563803460b is 50, key is test_row_0/A:col10/1731192102848/Put/seqid=0 2024-11-09T22:41:42,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742241_1417 (size=16681) 2024-11-09T22:41:42,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T22:41:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:42,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:42,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192162865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192162865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192162866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192162871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192162872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192162975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192162985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192162985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192162985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:42,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192162985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T22:41:43,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:43,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T22:41:43,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T22:41:43,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:43,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192163195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192163195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192163204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192163204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192163204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/5bc273063b7441f1af5da6563803460b 2024-11-09T22:41:43,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/aa4c71bac7bc449aaf93b15e1af2138f is 50, key is test_row_0/B:col10/1731192102848/Put/seqid=0 2024-11-09T22:41:43,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742242_1418 (size=12001) 2024-11-09T22:41:43,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T22:41:43,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:43,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T22:41:43,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T22:41:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192163505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192163505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192163515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192163515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192163515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:43,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T22:41:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:43,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/aa4c71bac7bc449aaf93b15e1af2138f 2024-11-09T22:41:43,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/230ad7673b674283a896c35fc47cd4ac is 50, key is test_row_0/C:col10/1731192102848/Put/seqid=0 2024-11-09T22:41:43,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742243_1419 (size=12001) 2024-11-09T22:41:43,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/230ad7673b674283a896c35fc47cd4ac 2024-11-09T22:41:43,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/5bc273063b7441f1af5da6563803460b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc273063b7441f1af5da6563803460b 2024-11-09T22:41:43,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc273063b7441f1af5da6563803460b, entries=250, sequenceid=116, filesize=16.3 K 2024-11-09T22:41:43,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/aa4c71bac7bc449aaf93b15e1af2138f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/aa4c71bac7bc449aaf93b15e1af2138f 2024-11-09T22:41:43,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/aa4c71bac7bc449aaf93b15e1af2138f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T22:41:43,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/230ad7673b674283a896c35fc47cd4ac as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/230ad7673b674283a896c35fc47cd4ac 2024-11-09T22:41:43,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/230ad7673b674283a896c35fc47cd4ac, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T22:41:43,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for bd15f4850dbe30ecfd33655486a40c35 in 846ms, sequenceid=116, compaction requested=true 2024-11-09T22:41:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:43,696 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:43,696 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:43,697 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:43,697 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:43,697 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/B is initiating minor compaction (all files) 2024-11-09T22:41:43,697 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/A is initiating minor compaction (all files) 2024-11-09T22:41:43,697 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/A in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,697 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/B in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,697 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/27c1a2ee967b417596a9686c92801600, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/86d2421bb2094667a025f7b773534699, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/1f14afa785294c3bb12a0d09564285d2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/aa4c71bac7bc449aaf93b15e1af2138f] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=47.0 K 2024-11-09T22:41:43,697 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/092c651b8f574023a0f8d8feb6b38f91, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/34983a4bfdfb4d9daa61f81133861fa5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/796490c1bcfc41a0bb21a21069bca0ce, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc273063b7441f1af5da6563803460b] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=53.8 K 2024-11-09T22:41:43,698 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c1a2ee967b417596a9686c92801600, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731192098652 2024-11-09T22:41:43,698 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 092c651b8f574023a0f8d8feb6b38f91, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731192098652 2024-11-09T22:41:43,698 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d2421bb2094667a025f7b773534699, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731192099317 2024-11-09T22:41:43,698 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34983a4bfdfb4d9daa61f81133861fa5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731192099317 2024-11-09T22:41:43,698 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f14afa785294c3bb12a0d09564285d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192100511 2024-11-09T22:41:43,698 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 796490c1bcfc41a0bb21a21069bca0ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192100511 2024-11-09T22:41:43,698 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting aa4c71bac7bc449aaf93b15e1af2138f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731192101697 2024-11-09T22:41:43,699 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc273063b7441f1af5da6563803460b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731192101687 2024-11-09T22:41:43,707 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#B#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:43,707 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#A#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:43,708 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/c1adba3b872446c7950d7902c980e48c is 50, key is test_row_0/B:col10/1731192102848/Put/seqid=0 2024-11-09T22:41:43,708 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/3796e2dc9a8f459d8bc852994f065473 is 50, key is test_row_0/A:col10/1731192102848/Put/seqid=0 2024-11-09T22:41:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742244_1420 (size=12241) 2024-11-09T22:41:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742245_1421 (size=12241) 2024-11-09T22:41:43,715 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/3796e2dc9a8f459d8bc852994f065473 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/3796e2dc9a8f459d8bc852994f065473 2024-11-09T22:41:43,718 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/A of bd15f4850dbe30ecfd33655486a40c35 into 3796e2dc9a8f459d8bc852994f065473(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:43,718 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:43,718 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/A, priority=12, startTime=1731192103696; duration=0sec 2024-11-09T22:41:43,719 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:43,719 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:A 2024-11-09T22:41:43,719 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:43,720 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:43,720 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/C is initiating minor compaction (all files) 2024-11-09T22:41:43,720 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/C in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,720 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/34c16c0e63da4b8ba51f597bf17176fc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/8b6b756f15504024871780b4e466d6bf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/a436dd45365840dfa99a186fc790bfaf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/230ad7673b674283a896c35fc47cd4ac] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=47.0 K 2024-11-09T22:41:43,720 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c16c0e63da4b8ba51f597bf17176fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731192098652 2024-11-09T22:41:43,720 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b6b756f15504024871780b4e466d6bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731192099317 2024-11-09T22:41:43,721 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a436dd45365840dfa99a186fc790bfaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731192100511 2024-11-09T22:41:43,721 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 230ad7673b674283a896c35fc47cd4ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731192101697 2024-11-09T22:41:43,728 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T22:41:43,728 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/1d2c26c5f98f46d2b9d94996ec3bc83c is 50, key is test_row_0/C:col10/1731192102848/Put/seqid=0 2024-11-09T22:41:43,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742246_1422 (size=12241) 2024-11-09T22:41:43,740 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/1d2c26c5f98f46d2b9d94996ec3bc83c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/1d2c26c5f98f46d2b9d94996ec3bc83c 2024-11-09T22:41:43,743 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/C of bd15f4850dbe30ecfd33655486a40c35 into 1d2c26c5f98f46d2b9d94996ec3bc83c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:43,743 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:43,744 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/C, priority=12, startTime=1731192103696; duration=0sec 2024-11-09T22:41:43,744 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:43,744 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:C 2024-11-09T22:41:43,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-09T22:41:43,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:43,776 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:41:43,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/fa66af81c2a64baca044fa60e2990267 is 50, key is test_row_0/A:col10/1731192102871/Put/seqid=0 2024-11-09T22:41:43,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742247_1423 (size=12001) 2024-11-09T22:41:43,815 DEBUG [regionserver/9d8c37854f6e:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-09T22:41:43,824 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8af89a87cc9425ebab946a2d8d343de, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1b759710b1ad44e7847e18985c9eaf56, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/445cb116b40d4187b44bfca04ceef26e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/092c651b8f574023a0f8d8feb6b38f91, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/34983a4bfdfb4d9daa61f81133861fa5, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/796490c1bcfc41a0bb21a21069bca0ce, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc273063b7441f1af5da6563803460b] to archive 2024-11-09T22:41:43,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:43,827 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8af89a87cc9425ebab946a2d8d343de to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8af89a87cc9425ebab946a2d8d343de 2024-11-09T22:41:43,828 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1b759710b1ad44e7847e18985c9eaf56 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1b759710b1ad44e7847e18985c9eaf56 2024-11-09T22:41:43,828 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/445cb116b40d4187b44bfca04ceef26e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/445cb116b40d4187b44bfca04ceef26e 2024-11-09T22:41:43,829 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/092c651b8f574023a0f8d8feb6b38f91 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/092c651b8f574023a0f8d8feb6b38f91 2024-11-09T22:41:43,830 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/34983a4bfdfb4d9daa61f81133861fa5 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/34983a4bfdfb4d9daa61f81133861fa5 2024-11-09T22:41:43,831 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/796490c1bcfc41a0bb21a21069bca0ce to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/796490c1bcfc41a0bb21a21069bca0ce 2024-11-09T22:41:43,832 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc273063b7441f1af5da6563803460b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc273063b7441f1af5da6563803460b 2024-11-09T22:41:43,833 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2c549bae678b4eea8eb224135665dd8f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d7b7cee47a23455d9dce7ded1dd2b8e4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/821790a688d443e0a0e0f23c8ffe6a31] to archive 2024-11-09T22:41:43,833 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:43,834 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2c549bae678b4eea8eb224135665dd8f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2c549bae678b4eea8eb224135665dd8f 2024-11-09T22:41:43,835 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d7b7cee47a23455d9dce7ded1dd2b8e4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d7b7cee47a23455d9dce7ded1dd2b8e4 2024-11-09T22:41:43,836 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/821790a688d443e0a0e0f23c8ffe6a31 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/821790a688d443e0a0e0f23c8ffe6a31 2024-11-09T22:41:43,837 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/ff0aad08f26e47aea0fc220464c68f95, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/cc4e51c632b24a3583a7db3013b2cf02, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/34c16c0e63da4b8ba51f597bf17176fc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/8b6b756f15504024871780b4e466d6bf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/a436dd45365840dfa99a186fc790bfaf, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/230ad7673b674283a896c35fc47cd4ac] to archive 2024-11-09T22:41:43,838 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:41:43,839 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/ff0aad08f26e47aea0fc220464c68f95 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/ff0aad08f26e47aea0fc220464c68f95 2024-11-09T22:41:43,840 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/cc4e51c632b24a3583a7db3013b2cf02 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/cc4e51c632b24a3583a7db3013b2cf02 2024-11-09T22:41:43,840 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/34c16c0e63da4b8ba51f597bf17176fc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/34c16c0e63da4b8ba51f597bf17176fc 2024-11-09T22:41:43,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d5c98bf16d9b4f14a7fbf54bcae0ecfd 2024-11-09T22:41:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T22:41:43,852 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/8b6b756f15504024871780b4e466d6bf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/8b6b756f15504024871780b4e466d6bf 2024-11-09T22:41:43,856 WARN [regionserver/9d8c37854f6e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 4, running: 1 2024-11-09T22:41:43,860 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/a436dd45365840dfa99a186fc790bfaf to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/a436dd45365840dfa99a186fc790bfaf 2024-11-09T22:41:43,861 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d8c37854f6e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/230ad7673b674283a896c35fc47cd4ac to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/230ad7673b674283a896c35fc47cd4ac 2024-11-09T22:41:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:44,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:44,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192164061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192164064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192164065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192164065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192164067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,115 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/c1adba3b872446c7950d7902c980e48c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c1adba3b872446c7950d7902c980e48c 2024-11-09T22:41:44,118 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/B of bd15f4850dbe30ecfd33655486a40c35 into c1adba3b872446c7950d7902c980e48c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:44,118 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:44,118 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/B, priority=12, startTime=1731192103696; duration=0sec 2024-11-09T22:41:44,119 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:44,119 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:B 2024-11-09T22:41:44,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192164175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192164175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,183 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/fa66af81c2a64baca044fa60e2990267 2024-11-09T22:41:44,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/62b2e24164844ad6bf73f66ef930199a is 50, key is test_row_0/B:col10/1731192102871/Put/seqid=0 2024-11-09T22:41:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192164185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192164186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192164186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742248_1424 (size=12001) 2024-11-09T22:41:44,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192164385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192164385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192164394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192164395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192164395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,595 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/62b2e24164844ad6bf73f66ef930199a 2024-11-09T22:41:44,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/d2c58585403b4b81ae51c30c2426be93 is 50, key is test_row_0/C:col10/1731192102871/Put/seqid=0 2024-11-09T22:41:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742249_1425 (size=12001) 2024-11-09T22:41:44,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192164695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192164695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192164705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192164705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192164705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:44,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T22:41:45,006 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/d2c58585403b4b81ae51c30c2426be93 2024-11-09T22:41:45,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/fa66af81c2a64baca044fa60e2990267 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/fa66af81c2a64baca044fa60e2990267 2024-11-09T22:41:45,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/fa66af81c2a64baca044fa60e2990267, entries=150, sequenceid=130, filesize=11.7 K 2024-11-09T22:41:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/62b2e24164844ad6bf73f66ef930199a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/62b2e24164844ad6bf73f66ef930199a 2024-11-09T22:41:45,017 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/62b2e24164844ad6bf73f66ef930199a, entries=150, sequenceid=130, filesize=11.7 K 2024-11-09T22:41:45,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/d2c58585403b4b81ae51c30c2426be93 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d2c58585403b4b81ae51c30c2426be93 2024-11-09T22:41:45,020 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d2c58585403b4b81ae51c30c2426be93, entries=150, sequenceid=130, filesize=11.7 K 2024-11-09T22:41:45,021 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bd15f4850dbe30ecfd33655486a40c35 in 1244ms, sequenceid=130, compaction requested=false 2024-11-09T22:41:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-09T22:41:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-09T22:41:45,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-09T22:41:45,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3140 sec 2024-11-09T22:41:45,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.3180 sec 2024-11-09T22:41:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:45,208 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:45,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:41:45,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/327a28eee8f6424c86a49c390602fef2 is 50, key is test_row_0/A:col10/1731192104065/Put/seqid=0 2024-11-09T22:41:45,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742250_1426 (size=14541) 2024-11-09T22:41:45,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192165221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192165222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192165223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192165225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192165225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192165335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192165335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192165335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192165335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192165544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192165545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192165545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192165545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/327a28eee8f6424c86a49c390602fef2 2024-11-09T22:41:45,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9575c0e303f84d98b5da6310e508917d is 50, key is test_row_0/B:col10/1731192104065/Put/seqid=0 2024-11-09T22:41:45,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742251_1427 (size=12151) 2024-11-09T22:41:45,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192165855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192165855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192165855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192165855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9575c0e303f84d98b5da6310e508917d 2024-11-09T22:41:46,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/b82862fd903a4034936a406a54c752d9 is 50, key is test_row_0/C:col10/1731192104065/Put/seqid=0 2024-11-09T22:41:46,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742252_1428 (size=12151) 2024-11-09T22:41:46,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/b82862fd903a4034936a406a54c752d9 2024-11-09T22:41:46,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/327a28eee8f6424c86a49c390602fef2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/327a28eee8f6424c86a49c390602fef2 2024-11-09T22:41:46,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/327a28eee8f6424c86a49c390602fef2, entries=200, sequenceid=156, filesize=14.2 K 2024-11-09T22:41:46,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9575c0e303f84d98b5da6310e508917d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9575c0e303f84d98b5da6310e508917d 2024-11-09T22:41:46,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9575c0e303f84d98b5da6310e508917d, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T22:41:46,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/b82862fd903a4034936a406a54c752d9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/b82862fd903a4034936a406a54c752d9 2024-11-09T22:41:46,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/b82862fd903a4034936a406a54c752d9, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T22:41:46,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for bd15f4850dbe30ecfd33655486a40c35 in 850ms, sequenceid=156, compaction requested=true 2024-11-09T22:41:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:46,058 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:46,058 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:46,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:46,059 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:46,059 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/B is initiating minor compaction (all files) 2024-11-09T22:41:46,059 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:46,059 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/A is initiating minor compaction (all files) 2024-11-09T22:41:46,059 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/B in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:46,059 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/A in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:46,059 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c1adba3b872446c7950d7902c980e48c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/62b2e24164844ad6bf73f66ef930199a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9575c0e303f84d98b5da6310e508917d] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=35.5 K 2024-11-09T22:41:46,059 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/3796e2dc9a8f459d8bc852994f065473, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/fa66af81c2a64baca044fa60e2990267, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/327a28eee8f6424c86a49c390602fef2] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=37.9 K 2024-11-09T22:41:46,060 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c1adba3b872446c7950d7902c980e48c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731192101697 2024-11-09T22:41:46,060 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3796e2dc9a8f459d8bc852994f065473, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731192101697 2024-11-09T22:41:46,060 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa66af81c2a64baca044fa60e2990267, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731192102865 2024-11-09T22:41:46,060 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 62b2e24164844ad6bf73f66ef930199a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731192102865 2024-11-09T22:41:46,060 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 327a28eee8f6424c86a49c390602fef2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731192104063 2024-11-09T22:41:46,060 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9575c0e303f84d98b5da6310e508917d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731192104063 2024-11-09T22:41:46,066 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:46,066 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#A#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:46,066 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/6329aa36d4864a48a4a096866fb32430 is 50, key is test_row_0/B:col10/1731192104065/Put/seqid=0 2024-11-09T22:41:46,066 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/96ab5588f61346e1bc5c172c7bb006dd is 50, key is test_row_0/A:col10/1731192104065/Put/seqid=0 2024-11-09T22:41:46,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742254_1430 (size=12391) 2024-11-09T22:41:46,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742253_1429 (size=12254) 2024-11-09T22:41:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:46,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:46,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:41:46,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a01516a1f2ac49c693f2c3183b13a74e is 50, key is test_row_0/A:col10/1731192105224/Put/seqid=0 2024-11-09T22:41:46,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742255_1431 (size=14541) 2024-11-09T22:41:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192166365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192166365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192166370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192166371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192166374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,473 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/6329aa36d4864a48a4a096866fb32430 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/6329aa36d4864a48a4a096866fb32430 2024-11-09T22:41:46,473 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/96ab5588f61346e1bc5c172c7bb006dd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/96ab5588f61346e1bc5c172c7bb006dd 2024-11-09T22:41:46,477 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/B of bd15f4850dbe30ecfd33655486a40c35 into 6329aa36d4864a48a4a096866fb32430(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:46,477 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:46,477 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/B, priority=13, startTime=1731192106058; duration=0sec 2024-11-09T22:41:46,477 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:46,477 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:B 2024-11-09T22:41:46,477 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:46,478 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:46,478 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/C is initiating minor compaction (all files) 2024-11-09T22:41:46,478 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/C in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:46,478 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/1d2c26c5f98f46d2b9d94996ec3bc83c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d2c58585403b4b81ae51c30c2426be93, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/b82862fd903a4034936a406a54c752d9] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=35.5 K 2024-11-09T22:41:46,479 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2c26c5f98f46d2b9d94996ec3bc83c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731192101697 2024-11-09T22:41:46,479 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d2c58585403b4b81ae51c30c2426be93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731192102865 2024-11-09T22:41:46,479 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting b82862fd903a4034936a406a54c752d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731192104063 2024-11-09T22:41:46,483 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/A of bd15f4850dbe30ecfd33655486a40c35 into 96ab5588f61346e1bc5c172c7bb006dd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:46,483 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:46,483 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/A, priority=13, startTime=1731192106058; duration=0sec 2024-11-09T22:41:46,483 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:46,483 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:A 2024-11-09T22:41:46,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192166484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192166484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,528 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:46,528 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/213b0c80f1724b4b947e481664ebe961 is 50, key is test_row_0/C:col10/1731192104065/Put/seqid=0 2024-11-09T22:41:46,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742256_1432 (size=12254) 2024-11-09T22:41:46,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a01516a1f2ac49c693f2c3183b13a74e 2024-11-09T22:41:46,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/ed214fffe8cf4c05945432460b6dd915 is 50, key is test_row_0/B:col10/1731192105224/Put/seqid=0 2024-11-09T22:41:46,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742257_1433 (size=12151) 2024-11-09T22:41:46,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/ed214fffe8cf4c05945432460b6dd915 2024-11-09T22:41:46,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/861fd11c5cca4af0818868362f9aeccc is 50, key is test_row_0/C:col10/1731192105224/Put/seqid=0 2024-11-09T22:41:46,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742258_1434 (size=12151) 2024-11-09T22:41:46,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192166695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192166695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-09T22:41:46,875 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:46,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-09T22:41:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T22:41:46,877 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:46,877 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:46,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:46,939 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/213b0c80f1724b4b947e481664ebe961 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/213b0c80f1724b4b947e481664ebe961 2024-11-09T22:41:46,943 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/C of bd15f4850dbe30ecfd33655486a40c35 into 213b0c80f1724b4b947e481664ebe961(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:46,943 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:46,943 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/C, priority=13, startTime=1731192106058; duration=0sec 2024-11-09T22:41:46,943 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:46,943 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:C 2024-11-09T22:41:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T22:41:47,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192167004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192167015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T22:41:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:47,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:47,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/861fd11c5cca4af0818868362f9aeccc 2024-11-09T22:41:47,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a01516a1f2ac49c693f2c3183b13a74e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a01516a1f2ac49c693f2c3183b13a74e 2024-11-09T22:41:47,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a01516a1f2ac49c693f2c3183b13a74e, entries=200, sequenceid=168, filesize=14.2 K 2024-11-09T22:41:47,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/ed214fffe8cf4c05945432460b6dd915 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/ed214fffe8cf4c05945432460b6dd915 2024-11-09T22:41:47,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/ed214fffe8cf4c05945432460b6dd915, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T22:41:47,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/861fd11c5cca4af0818868362f9aeccc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/861fd11c5cca4af0818868362f9aeccc 2024-11-09T22:41:47,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/861fd11c5cca4af0818868362f9aeccc, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T22:41:47,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bd15f4850dbe30ecfd33655486a40c35 in 844ms, sequenceid=168, compaction requested=false 2024-11-09T22:41:47,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:47,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T22:41:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:47,182 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T22:41:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T22:41:47,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/b717030cde9e4808aab09f23f1ad8087 is 50, key is test_row_0/A:col10/1731192106370/Put/seqid=0 2024-11-09T22:41:47,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742259_1435 (size=12151) 2024-11-09T22:41:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192167402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192167408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192167409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T22:41:47,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192167515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192167515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192167524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192167524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192167525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,602 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/b717030cde9e4808aab09f23f1ad8087 2024-11-09T22:41:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/06d1356043904834b3a7cd8e854d4960 is 50, key is test_row_0/B:col10/1731192106370/Put/seqid=0 2024-11-09T22:41:47,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742260_1436 (size=12151) 2024-11-09T22:41:47,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192167725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192167734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192167734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,011 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/06d1356043904834b3a7cd8e854d4960 2024-11-09T22:41:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T22:41:48,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/c409c31575214ce69371f33f3230e606 is 50, key is test_row_0/C:col10/1731192106370/Put/seqid=0 2024-11-09T22:41:48,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742261_1437 (size=12151) 2024-11-09T22:41:48,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192168045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192168045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192168055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,421 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/c409c31575214ce69371f33f3230e606 2024-11-09T22:41:48,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/b717030cde9e4808aab09f23f1ad8087 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b717030cde9e4808aab09f23f1ad8087 2024-11-09T22:41:48,428 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b717030cde9e4808aab09f23f1ad8087, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T22:41:48,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/06d1356043904834b3a7cd8e854d4960 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/06d1356043904834b3a7cd8e854d4960 2024-11-09T22:41:48,432 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/06d1356043904834b3a7cd8e854d4960, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T22:41:48,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/c409c31575214ce69371f33f3230e606 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c409c31575214ce69371f33f3230e606 2024-11-09T22:41:48,436 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c409c31575214ce69371f33f3230e606, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T22:41:48,437 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for bd15f4850dbe30ecfd33655486a40c35 in 1255ms, sequenceid=195, compaction requested=true 2024-11-09T22:41:48,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:48,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:48,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-09T22:41:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-09T22:41:48,439 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-09T22:41:48,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5610 sec 2024-11-09T22:41:48,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5650 sec 2024-11-09T22:41:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:48,541 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:48,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T22:41:48,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/50022f14da1745cbbe6ef982a3494bdb is 50, key is test_row_0/A:col10/1731192107407/Put/seqid=0 2024-11-09T22:41:48,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742262_1438 (size=14541) 2024-11-09T22:41:48,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192168598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192168599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192168600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192168601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192168605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192168714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192168715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192168715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192168715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192168725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192168925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192168925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192168925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192168925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/50022f14da1745cbbe6ef982a3494bdb 2024-11-09T22:41:48,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192168945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:48,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/fbafa7e09dc84e768f23a5501f34dc99 is 50, key is test_row_0/B:col10/1731192107407/Put/seqid=0 2024-11-09T22:41:48,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742263_1439 (size=12151) 2024-11-09T22:41:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T22:41:49,035 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:49,036 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-09T22:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T22:41:49,037 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:49,038 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:49,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T22:41:49,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:49,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192169235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192169235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192169235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192169245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192169264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T22:41:49,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/fbafa7e09dc84e768f23a5501f34dc99 2024-11-09T22:41:49,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0f10be6ec3c74e6ca310da4bb8077c7c is 50, key is test_row_0/C:col10/1731192107407/Put/seqid=0 2024-11-09T22:41:49,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742264_1440 (size=12151) 2024-11-09T22:41:49,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:49,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:49,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T22:41:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192169755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192169755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192169755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0f10be6ec3c74e6ca310da4bb8077c7c 2024-11-09T22:41:49,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/50022f14da1745cbbe6ef982a3494bdb as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/50022f14da1745cbbe6ef982a3494bdb 2024-11-09T22:41:49,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192169764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/50022f14da1745cbbe6ef982a3494bdb, entries=200, sequenceid=208, filesize=14.2 K 2024-11-09T22:41:49,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/fbafa7e09dc84e768f23a5501f34dc99 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fbafa7e09dc84e768f23a5501f34dc99 2024-11-09T22:41:49,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fbafa7e09dc84e768f23a5501f34dc99, entries=150, sequenceid=208, filesize=11.9 K 2024-11-09T22:41:49,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0f10be6ec3c74e6ca310da4bb8077c7c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0f10be6ec3c74e6ca310da4bb8077c7c 2024-11-09T22:41:49,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0f10be6ec3c74e6ca310da4bb8077c7c, entries=150, sequenceid=208, filesize=11.9 K 2024-11-09T22:41:49,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for bd15f4850dbe30ecfd33655486a40c35 in 1241ms, sequenceid=208, compaction requested=true 2024-11-09T22:41:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:49,782 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:49,783 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:49,784 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:49,784 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53487 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:49,784 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/A is initiating minor compaction (all files) 2024-11-09T22:41:49,784 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/B is initiating minor compaction (all files) 2024-11-09T22:41:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:49,784 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/B in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,784 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/A in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,784 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:49,784 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/6329aa36d4864a48a4a096866fb32430, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/ed214fffe8cf4c05945432460b6dd915, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/06d1356043904834b3a7cd8e854d4960, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fbafa7e09dc84e768f23a5501f34dc99] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=47.7 K 2024-11-09T22:41:49,784 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/96ab5588f61346e1bc5c172c7bb006dd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a01516a1f2ac49c693f2c3183b13a74e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b717030cde9e4808aab09f23f1ad8087, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/50022f14da1745cbbe6ef982a3494bdb] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=52.2 K 2024-11-09T22:41:49,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T22:41:49,785 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96ab5588f61346e1bc5c172c7bb006dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731192104063 2024-11-09T22:41:49,785 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6329aa36d4864a48a4a096866fb32430, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731192104063 2024-11-09T22:41:49,785 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a01516a1f2ac49c693f2c3183b13a74e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731192105220 2024-11-09T22:41:49,785 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ed214fffe8cf4c05945432460b6dd915, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731192105220 2024-11-09T22:41:49,786 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b717030cde9e4808aab09f23f1ad8087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731192106310 2024-11-09T22:41:49,786 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 06d1356043904834b3a7cd8e854d4960, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731192106310 2024-11-09T22:41:49,786 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50022f14da1745cbbe6ef982a3494bdb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731192107400 2024-11-09T22:41:49,786 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting fbafa7e09dc84e768f23a5501f34dc99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731192107406 2024-11-09T22:41:49,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/0de6d3dc320d4b5db41d2b75a00e15dc is 50, key is test_row_0/A:col10/1731192108603/Put/seqid=0 2024-11-09T22:41:49,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:49,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:49,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,804 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:49,805 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/2dc81729fecd442bbbd44548621e62c4 is 50, key is test_row_0/B:col10/1731192107407/Put/seqid=0 2024-11-09T22:41:49,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742265_1441 (size=14541) 2024-11-09T22:41:49,807 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:49,808 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/1f077a6710c746e29c4695c2b58acc4e is 50, key is test_row_0/A:col10/1731192107407/Put/seqid=0 2024-11-09T22:41:49,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/0de6d3dc320d4b5db41d2b75a00e15dc 2024-11-09T22:41:49,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742266_1442 (size=12527) 2024-11-09T22:41:49,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/fc4b5baa4e1e491c819611f90ba8eb02 is 50, key is test_row_0/B:col10/1731192108603/Put/seqid=0 2024-11-09T22:41:49,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742267_1443 (size=12391) 2024-11-09T22:41:49,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742268_1444 (size=12151) 2024-11-09T22:41:49,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/fc4b5baa4e1e491c819611f90ba8eb02 2024-11-09T22:41:49,830 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/2dc81729fecd442bbbd44548621e62c4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2dc81729fecd442bbbd44548621e62c4 2024-11-09T22:41:49,834 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/B of bd15f4850dbe30ecfd33655486a40c35 into 2dc81729fecd442bbbd44548621e62c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:49,834 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:49,834 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/B, priority=12, startTime=1731192109782; duration=0sec 2024-11-09T22:41:49,835 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:49,835 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:B 2024-11-09T22:41:49,835 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:49,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/14850a92c53647e8ae964e681b920f40 is 50, key is test_row_0/C:col10/1731192108603/Put/seqid=0 2024-11-09T22:41:49,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48707 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:49,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/C is initiating minor compaction (all files) 2024-11-09T22:41:49,837 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/C in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,837 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/213b0c80f1724b4b947e481664ebe961, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/861fd11c5cca4af0818868362f9aeccc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c409c31575214ce69371f33f3230e606, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0f10be6ec3c74e6ca310da4bb8077c7c] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=47.6 K 2024-11-09T22:41:49,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 213b0c80f1724b4b947e481664ebe961, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731192104063 2024-11-09T22:41:49,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 861fd11c5cca4af0818868362f9aeccc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731192105220 2024-11-09T22:41:49,838 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c409c31575214ce69371f33f3230e606, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731192106310 2024-11-09T22:41:49,838 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f10be6ec3c74e6ca310da4bb8077c7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731192107406 2024-11-09T22:41:49,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742269_1445 (size=12151) 2024-11-09T22:41:49,845 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#C#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:49,846 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/c5ba6a2c4bf344c093faae753b8bf2a7 is 50, key is test_row_0/C:col10/1731192107407/Put/seqid=0 2024-11-09T22:41:49,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742270_1446 (size=12391) 2024-11-09T22:41:49,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192169867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:49,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:49,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:49,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:49,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:49,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192169975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:50,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:50,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T22:41:50,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192170195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:50,232 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/1f077a6710c746e29c4695c2b58acc4e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1f077a6710c746e29c4695c2b58acc4e 2024-11-09T22:41:50,236 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/A of bd15f4850dbe30ecfd33655486a40c35 into 1f077a6710c746e29c4695c2b58acc4e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:50,236 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:50,236 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/A, priority=12, startTime=1731192109782; duration=0sec 2024-11-09T22:41:50,236 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:50,236 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:A 2024-11-09T22:41:50,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/14850a92c53647e8ae964e681b920f40 2024-11-09T22:41:50,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/0de6d3dc320d4b5db41d2b75a00e15dc as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/0de6d3dc320d4b5db41d2b75a00e15dc 2024-11-09T22:41:50,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/0de6d3dc320d4b5db41d2b75a00e15dc, entries=200, sequenceid=232, filesize=14.2 K 2024-11-09T22:41:50,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/fc4b5baa4e1e491c819611f90ba8eb02 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fc4b5baa4e1e491c819611f90ba8eb02 2024-11-09T22:41:50,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fc4b5baa4e1e491c819611f90ba8eb02, entries=150, sequenceid=232, filesize=11.9 K 2024-11-09T22:41:50,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/14850a92c53647e8ae964e681b920f40 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/14850a92c53647e8ae964e681b920f40 2024-11-09T22:41:50,259 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/c5ba6a2c4bf344c093faae753b8bf2a7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c5ba6a2c4bf344c093faae753b8bf2a7 2024-11-09T22:41:50,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:50,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/14850a92c53647e8ae964e681b920f40, entries=150, sequenceid=232, filesize=11.9 K 2024-11-09T22:41:50,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:50,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:50,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:50,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for bd15f4850dbe30ecfd33655486a40c35 in 477ms, sequenceid=232, compaction requested=false 2024-11-09T22:41:50,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:50,264 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/C of bd15f4850dbe30ecfd33655486a40c35 into c5ba6a2c4bf344c093faae753b8bf2a7(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:50,264 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:50,264 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/C, priority=12, startTime=1731192109783; duration=0sec 2024-11-09T22:41:50,264 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:50,264 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:C 2024-11-09T22:41:50,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T22:41:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:50,413 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:41:50,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/c919f4368a1d420b926f2f0473461309 is 50, key is test_row_0/A:col10/1731192109842/Put/seqid=0 2024-11-09T22:41:50,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742271_1447 (size=12151) 2024-11-09T22:41:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:50,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192170643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:50,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192170755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:50,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192170765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:50,769 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in bd15f4850dbe30ecfd33655486a40c35 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:50,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192170774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:50,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192170775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:50,779 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in bd15f4850dbe30ecfd33655486a40c35 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:50,779 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in bd15f4850dbe30ecfd33655486a40c35 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192170785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:50,789 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_2' in bd15f4850dbe30ecfd33655486a40c35 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:50,820 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/c919f4368a1d420b926f2f0473461309 2024-11-09T22:41:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/596726ac037845f6be37f6be5c936f4c is 50, key is test_row_0/B:col10/1731192109842/Put/seqid=0 2024-11-09T22:41:50,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742272_1448 (size=12151) 2024-11-09T22:41:50,831 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/596726ac037845f6be37f6be5c936f4c 2024-11-09T22:41:50,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/832bed9677194ed695a9c06aa6d53cc9 is 50, key is test_row_0/C:col10/1731192109842/Put/seqid=0 2024-11-09T22:41:50,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742273_1449 (size=12151) 2024-11-09T22:41:50,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192170965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T22:41:51,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/832bed9677194ed695a9c06aa6d53cc9 2024-11-09T22:41:51,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/c919f4368a1d420b926f2f0473461309 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/c919f4368a1d420b926f2f0473461309 2024-11-09T22:41:51,246 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/c919f4368a1d420b926f2f0473461309, entries=150, sequenceid=247, filesize=11.9 K 2024-11-09T22:41:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/596726ac037845f6be37f6be5c936f4c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/596726ac037845f6be37f6be5c936f4c 2024-11-09T22:41:51,249 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/596726ac037845f6be37f6be5c936f4c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-09T22:41:51,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/832bed9677194ed695a9c06aa6d53cc9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/832bed9677194ed695a9c06aa6d53cc9 2024-11-09T22:41:51,252 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/832bed9677194ed695a9c06aa6d53cc9, entries=150, sequenceid=247, filesize=11.9 K 2024-11-09T22:41:51,253 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bd15f4850dbe30ecfd33655486a40c35 in 840ms, sequenceid=247, compaction requested=true 2024-11-09T22:41:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-09T22:41:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-09T22:41:51,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-09T22:41:51,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-11-09T22:41:51,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.2200 sec 2024-11-09T22:41:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:51,280 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:51,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:41:51,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/5bc44416396741e7a99ec377fe30ea18 is 50, key is test_row_0/A:col10/1731192111279/Put/seqid=0 2024-11-09T22:41:51,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742274_1450 (size=14741) 2024-11-09T22:41:51,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192171332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192171435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:51,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192171644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:51,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/5bc44416396741e7a99ec377fe30ea18 2024-11-09T22:41:51,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/bf1408e811d54d4ab78b626c49355e60 is 50, key is test_row_0/B:col10/1731192111279/Put/seqid=0 2024-11-09T22:41:51,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742275_1451 (size=12301) 2024-11-09T22:41:51,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192171965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/bf1408e811d54d4ab78b626c49355e60 2024-11-09T22:41:52,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0d3085de0ad64609b5e920e3e2347c23 is 50, key is test_row_0/C:col10/1731192111279/Put/seqid=0 2024-11-09T22:41:52,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742276_1452 (size=12301) 2024-11-09T22:41:52,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192172484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0d3085de0ad64609b5e920e3e2347c23 2024-11-09T22:41:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/5bc44416396741e7a99ec377fe30ea18 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc44416396741e7a99ec377fe30ea18 2024-11-09T22:41:52,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc44416396741e7a99ec377fe30ea18, entries=200, sequenceid=272, filesize=14.4 K 2024-11-09T22:41:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/bf1408e811d54d4ab78b626c49355e60 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/bf1408e811d54d4ab78b626c49355e60 2024-11-09T22:41:52,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/bf1408e811d54d4ab78b626c49355e60, entries=150, sequenceid=272, filesize=12.0 K 2024-11-09T22:41:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0d3085de0ad64609b5e920e3e2347c23 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d3085de0ad64609b5e920e3e2347c23 2024-11-09T22:41:52,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d3085de0ad64609b5e920e3e2347c23, entries=150, sequenceid=272, filesize=12.0 K 2024-11-09T22:41:52,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for bd15f4850dbe30ecfd33655486a40c35 in 1284ms, sequenceid=272, compaction requested=true 2024-11-09T22:41:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:52,564 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:52,564 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:52,565 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53824 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:52,565 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:52,565 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/A is initiating minor compaction (all files) 2024-11-09T22:41:52,565 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/B is initiating minor compaction (all files) 2024-11-09T22:41:52,565 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/A in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:52,565 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/B in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:52,565 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1f077a6710c746e29c4695c2b58acc4e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/0de6d3dc320d4b5db41d2b75a00e15dc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/c919f4368a1d420b926f2f0473461309, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc44416396741e7a99ec377fe30ea18] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=52.6 K 2024-11-09T22:41:52,565 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2dc81729fecd442bbbd44548621e62c4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fc4b5baa4e1e491c819611f90ba8eb02, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/596726ac037845f6be37f6be5c936f4c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/bf1408e811d54d4ab78b626c49355e60] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=48.0 K 2024-11-09T22:41:52,565 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f077a6710c746e29c4695c2b58acc4e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731192107406 2024-11-09T22:41:52,565 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dc81729fecd442bbbd44548621e62c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731192107406 2024-11-09T22:41:52,565 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de6d3dc320d4b5db41d2b75a00e15dc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731192108598 2024-11-09T22:41:52,566 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4b5baa4e1e491c819611f90ba8eb02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731192108600 2024-11-09T22:41:52,566 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c919f4368a1d420b926f2f0473461309, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731192109842 2024-11-09T22:41:52,566 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 596726ac037845f6be37f6be5c936f4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731192109842 2024-11-09T22:41:52,566 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bc44416396741e7a99ec377fe30ea18, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731192110615 2024-11-09T22:41:52,566 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1408e811d54d4ab78b626c49355e60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731192110615 2024-11-09T22:41:52,577 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#B#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:52,577 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:52,578 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/02efd0975362489d8aa11de64df4b9ee is 50, key is test_row_0/B:col10/1731192111279/Put/seqid=0 2024-11-09T22:41:52,578 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/1037633a1c714c339c0e7747b9e1cf6c is 50, key is test_row_0/A:col10/1731192111279/Put/seqid=0 2024-11-09T22:41:52,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742278_1454 (size=12677) 2024-11-09T22:41:52,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742277_1453 (size=12813) 2024-11-09T22:41:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:52,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:41:52,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/b8ac68d320a34f20b413e71c22d646bd is 50, key is test_row_0/A:col10/1731192112792/Put/seqid=0 2024-11-09T22:41:52,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742279_1455 (size=12301) 2024-11-09T22:41:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192172864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192172865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192172872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192172875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,987 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/1037633a1c714c339c0e7747b9e1cf6c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1037633a1c714c339c0e7747b9e1cf6c 2024-11-09T22:41:52,991 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/A of bd15f4850dbe30ecfd33655486a40c35 into 1037633a1c714c339c0e7747b9e1cf6c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:52,991 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:52,991 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/A, priority=12, startTime=1731192112564; duration=0sec 2024-11-09T22:41:52,991 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:52,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:52,991 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:A 2024-11-09T22:41:52,991 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:41:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192172985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:52,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192172985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,992 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/02efd0975362489d8aa11de64df4b9ee as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/02efd0975362489d8aa11de64df4b9ee 2024-11-09T22:41:52,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192172985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:52,992 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48994 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:41:52,993 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/C is initiating minor compaction (all files) 2024-11-09T22:41:52,993 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/C in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:52,993 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c5ba6a2c4bf344c093faae753b8bf2a7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/14850a92c53647e8ae964e681b920f40, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/832bed9677194ed695a9c06aa6d53cc9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d3085de0ad64609b5e920e3e2347c23] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=47.8 K 2024-11-09T22:41:52,993 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ba6a2c4bf344c093faae753b8bf2a7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731192107406 2024-11-09T22:41:52,993 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14850a92c53647e8ae964e681b920f40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731192108600 2024-11-09T22:41:52,993 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 832bed9677194ed695a9c06aa6d53cc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731192109842 2024-11-09T22:41:52,994 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d3085de0ad64609b5e920e3e2347c23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731192110615 2024-11-09T22:41:52,996 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/B of bd15f4850dbe30ecfd33655486a40c35 into 02efd0975362489d8aa11de64df4b9ee(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:52,996 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:52,996 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/B, priority=12, startTime=1731192112564; duration=0sec 2024-11-09T22:41:52,996 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:52,996 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:B 2024-11-09T22:41:53,000 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#C#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:53,000 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/79440134982743b1ac85c9887120932e is 50, key is test_row_0/C:col10/1731192111279/Put/seqid=0 2024-11-09T22:41:53,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742280_1456 (size=12677) 2024-11-09T22:41:53,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192172994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T22:41:53,195 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:53,196 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:41:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-09T22:41:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T22:41:53,197 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:41:53,198 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:41:53,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:41:53,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192173195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/b8ac68d320a34f20b413e71c22d646bd 2024-11-09T22:41:53,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9b0320abe86a43eea677e3758af3198a is 50, key is test_row_0/B:col10/1731192112792/Put/seqid=0 2024-11-09T22:41:53,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192173195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192173195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742281_1457 (size=12301) 2024-11-09T22:41:53,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192173215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T22:41:53,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:53,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:53,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,407 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/79440134982743b1ac85c9887120932e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/79440134982743b1ac85c9887120932e 2024-11-09T22:41:53,410 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/C of bd15f4850dbe30ecfd33655486a40c35 into 79440134982743b1ac85c9887120932e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:53,411 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:53,411 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/C, priority=12, startTime=1731192112564; duration=0sec 2024-11-09T22:41:53,411 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:53,411 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:C 2024-11-09T22:41:53,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192173505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192173505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,512 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in bd15f4850dbe30ecfd33655486a40c35 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T22:41:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192173514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192173514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192173535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:53,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9b0320abe86a43eea677e3758af3198a 2024-11-09T22:41:53,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/bf374ba0de274d20b7a350ab0f7b159f is 50, key is test_row_0/C:col10/1731192112792/Put/seqid=0 2024-11-09T22:41:53,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742282_1458 (size=12301) 2024-11-09T22:41:53,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:53,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T22:41:53,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:53,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/bf374ba0de274d20b7a350ab0f7b159f 2024-11-09T22:41:54,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/b8ac68d320a34f20b413e71c22d646bd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b8ac68d320a34f20b413e71c22d646bd 2024-11-09T22:41:54,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b8ac68d320a34f20b413e71c22d646bd, entries=150, sequenceid=284, filesize=12.0 K 2024-11-09T22:41:54,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9b0320abe86a43eea677e3758af3198a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9b0320abe86a43eea677e3758af3198a 2024-11-09T22:41:54,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:54,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192174024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192174025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:54,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9b0320abe86a43eea677e3758af3198a, entries=150, sequenceid=284, filesize=12.0 K 2024-11-09T22:41:54,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/bf374ba0de274d20b7a350ab0f7b159f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/bf374ba0de274d20b7a350ab0f7b159f 2024-11-09T22:41:54,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192174035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:54,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:54,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/bf374ba0de274d20b7a350ab0f7b159f, entries=150, sequenceid=284, filesize=12.0 K 2024-11-09T22:41:54,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bd15f4850dbe30ecfd33655486a40c35 in 1245ms, sequenceid=284, compaction requested=false 2024-11-09T22:41:54,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:54,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:41:54,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/4c0cef9a1d654108bccba4852cbc13f0 is 50, key is test_row_0/A:col10/1731192112864/Put/seqid=0 2024-11-09T22:41:54,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742283_1459 (size=14741) 2024-11-09T22:41:54,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192174089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:54,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:54,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:54,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192174195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:54,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T22:41:54,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192174405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:54,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/4c0cef9a1d654108bccba4852cbc13f0 2024-11-09T22:41:54,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/c652e90112f14201a45f51e343b65a5c is 50, key is test_row_0/B:col10/1731192112864/Put/seqid=0 2024-11-09T22:41:54,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742284_1460 (size=12301) 2024-11-09T22:41:54,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/c652e90112f14201a45f51e343b65a5c 2024-11-09T22:41:54,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/9838feb3cd3249b4b2f2d3db88f3f13c is 50, key is test_row_0/C:col10/1731192112864/Put/seqid=0 2024-11-09T22:41:54,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742285_1461 (size=12301) 2024-11-09T22:41:54,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192174725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:54,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:54,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:41:54,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/9838feb3cd3249b4b2f2d3db88f3f13c 2024-11-09T22:41:54,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/4c0cef9a1d654108bccba4852cbc13f0 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/4c0cef9a1d654108bccba4852cbc13f0 2024-11-09T22:41:54,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/4c0cef9a1d654108bccba4852cbc13f0, entries=200, sequenceid=312, filesize=14.4 K 2024-11-09T22:41:54,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/c652e90112f14201a45f51e343b65a5c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c652e90112f14201a45f51e343b65a5c 2024-11-09T22:41:54,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c652e90112f14201a45f51e343b65a5c, entries=150, sequenceid=312, filesize=12.0 K 2024-11-09T22:41:54,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/9838feb3cd3249b4b2f2d3db88f3f13c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/9838feb3cd3249b4b2f2d3db88f3f13c 2024-11-09T22:41:54,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/9838feb3cd3249b4b2f2d3db88f3f13c, entries=150, sequenceid=312, filesize=12.0 K 2024-11-09T22:41:54,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for bd15f4850dbe30ecfd33655486a40c35 in 846ms, sequenceid=312, compaction requested=true 2024-11-09T22:41:54,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:54,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:54,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:54,897 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:54,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:54,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:54,897 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:54,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:54,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:54,897 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:54,897 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:54,897 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/A is initiating minor compaction (all files) 2024-11-09T22:41:54,897 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/B is initiating minor compaction (all files) 2024-11-09T22:41:54,897 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/A in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,897 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/B in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:54,897 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1037633a1c714c339c0e7747b9e1cf6c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b8ac68d320a34f20b413e71c22d646bd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/4c0cef9a1d654108bccba4852cbc13f0] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=38.8 K 2024-11-09T22:41:54,897 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/02efd0975362489d8aa11de64df4b9ee, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9b0320abe86a43eea677e3758af3198a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c652e90112f14201a45f51e343b65a5c] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=36.5 K 2024-11-09T22:41:54,898 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1037633a1c714c339c0e7747b9e1cf6c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731192110615 2024-11-09T22:41:54,898 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 02efd0975362489d8aa11de64df4b9ee, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731192110615 2024-11-09T22:41:54,898 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b0320abe86a43eea677e3758af3198a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731192111314 2024-11-09T22:41:54,898 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8ac68d320a34f20b413e71c22d646bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731192111314 2024-11-09T22:41:54,898 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c652e90112f14201a45f51e343b65a5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731192112864 2024-11-09T22:41:54,898 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0cef9a1d654108bccba4852cbc13f0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731192112864 2024-11-09T22:41:54,903 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#B#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:54,904 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/cd69e7d93cda4845a6eecb3329d3913e is 50, key is test_row_0/B:col10/1731192112864/Put/seqid=0 2024-11-09T22:41:54,905 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:54,905 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/cb7061f6385e4e879fdeb4a5492145eb is 50, key is test_row_0/A:col10/1731192112864/Put/seqid=0 2024-11-09T22:41:54,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742286_1462 (size=12915) 2024-11-09T22:41:54,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742287_1463 (size=12779) 2024-11-09T22:41:55,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T22:41:55,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:55,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:41:55,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a8fe8d54d58a4d05a9e23c9300941286 is 50, key is test_row_0/A:col10/1731192114088/Put/seqid=0 2024-11-09T22:41:55,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742288_1464 (size=9857) 2024-11-09T22:41:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. as already flushing 2024-11-09T22:41:55,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192175096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192175097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192175104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192175215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192175215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192175215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192175235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,312 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/cd69e7d93cda4845a6eecb3329d3913e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/cd69e7d93cda4845a6eecb3329d3913e 2024-11-09T22:41:55,314 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/cb7061f6385e4e879fdeb4a5492145eb as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/cb7061f6385e4e879fdeb4a5492145eb 2024-11-09T22:41:55,316 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/B of bd15f4850dbe30ecfd33655486a40c35 into cd69e7d93cda4845a6eecb3329d3913e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:55,316 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:55,316 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/B, priority=13, startTime=1731192114897; duration=0sec 2024-11-09T22:41:55,316 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:55,316 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:B 2024-11-09T22:41:55,316 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:55,318 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:55,318 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/C is initiating minor compaction (all files) 2024-11-09T22:41:55,318 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/C in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:55,318 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/79440134982743b1ac85c9887120932e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/bf374ba0de274d20b7a350ab0f7b159f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/9838feb3cd3249b4b2f2d3db88f3f13c] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=36.4 K 2024-11-09T22:41:55,318 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 79440134982743b1ac85c9887120932e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731192110615 2024-11-09T22:41:55,319 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/A of bd15f4850dbe30ecfd33655486a40c35 into cb7061f6385e4e879fdeb4a5492145eb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:55,319 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:55,319 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/A, priority=13, startTime=1731192114896; duration=0sec 2024-11-09T22:41:55,319 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting bf374ba0de274d20b7a350ab0f7b159f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731192111314 2024-11-09T22:41:55,319 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:55,319 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:A 2024-11-09T22:41:55,319 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9838feb3cd3249b4b2f2d3db88f3f13c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731192112864 2024-11-09T22:41:55,324 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:55,325 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/f1804b8154cb4d7cb1f38423ef9e50cd is 50, key is test_row_0/C:col10/1731192112864/Put/seqid=0 2024-11-09T22:41:55,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742289_1465 (size=12779) 2024-11-09T22:41:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T22:41:55,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192175425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192175425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192175425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a8fe8d54d58a4d05a9e23c9300941286 2024-11-09T22:41:55,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/301626c179924a00887bf3a5109b924e is 50, key is test_row_0/B:col10/1731192114088/Put/seqid=0 2024-11-09T22:41:55,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742290_1466 (size=9857) 2024-11-09T22:41:55,448 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/301626c179924a00887bf3a5109b924e 2024-11-09T22:41:55,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/e5a910d0ba1241fdad19fd32c094f3ae is 50, key is test_row_0/C:col10/1731192114088/Put/seqid=0 2024-11-09T22:41:55,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742291_1467 (size=9857) 2024-11-09T22:41:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47012 deadline: 1731192175535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,539 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in bd15f4850dbe30ecfd33655486a40c35 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,735 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/f1804b8154cb4d7cb1f38423ef9e50cd as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/f1804b8154cb4d7cb1f38423ef9e50cd 2024-11-09T22:41:55,739 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/C of bd15f4850dbe30ecfd33655486a40c35 into f1804b8154cb4d7cb1f38423ef9e50cd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:55,739 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:55,739 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/C, priority=13, startTime=1731192114897; duration=0sec 2024-11-09T22:41:55,739 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:55,739 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:C 2024-11-09T22:41:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192175735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192175735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192175735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:55,857 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/e5a910d0ba1241fdad19fd32c094f3ae 2024-11-09T22:41:55,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/a8fe8d54d58a4d05a9e23c9300941286 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8fe8d54d58a4d05a9e23c9300941286 2024-11-09T22:41:55,866 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8fe8d54d58a4d05a9e23c9300941286, entries=100, sequenceid=323, filesize=9.6 K 2024-11-09T22:41:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/301626c179924a00887bf3a5109b924e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/301626c179924a00887bf3a5109b924e 2024-11-09T22:41:55,870 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/301626c179924a00887bf3a5109b924e, entries=100, sequenceid=323, filesize=9.6 K 2024-11-09T22:41:55,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/e5a910d0ba1241fdad19fd32c094f3ae as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/e5a910d0ba1241fdad19fd32c094f3ae 2024-11-09T22:41:55,873 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/e5a910d0ba1241fdad19fd32c094f3ae, entries=100, sequenceid=323, filesize=9.6 K 2024-11-09T22:41:55,874 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for bd15f4850dbe30ecfd33655486a40c35 in 846ms, sequenceid=323, compaction requested=false 2024-11-09T22:41:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-09T22:41:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-09T22:41:55,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-09T22:41:55,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6770 sec 2024-11-09T22:41:55,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.6810 sec 2024-11-09T22:41:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:56,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:56,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T22:41:56,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/d6561e66af31496eb681af64f15909b4 is 50, key is test_row_0/A:col10/1731192115102/Put/seqid=0 2024-11-09T22:41:56,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742292_1468 (size=14741) 2024-11-09T22:41:56,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192176257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192176261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192176262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192176263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,364 INFO [Thread-1818 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1818. 2024-11-09T22:41:56,364 DEBUG [Thread-1818 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:56,364 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,364 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:56,366 INFO [Thread-1814 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1814. 2024-11-09T22:41:56,366 DEBUG [Thread-1814 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:56,366 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,366 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:56,369 INFO [Thread-1816 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1816. 2024-11-09T22:41:56,370 DEBUG [Thread-1816 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:56,370 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,370 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:56,373 INFO [Thread-1820 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1820. 2024-11-09T22:41:56,374 DEBUG [Thread-1820 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:56,374 DEBUG [Thread-1820 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,374 DEBUG [Thread-1820 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:56,374 INFO [Thread-1822 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1822. 2024-11-09T22:41:56,374 DEBUG [Thread-1822 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:56,374 DEBUG [Thread-1822 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,374 DEBUG [Thread-1822 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:56,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192176375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192176375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192176375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192176376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192176586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192176586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192176587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192176587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/d6561e66af31496eb681af64f15909b4 2024-11-09T22:41:56,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9e08ec608309454eac77b7db958d54c7 is 50, key is test_row_0/B:col10/1731192115102/Put/seqid=0 2024-11-09T22:41:56,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742293_1469 (size=12301) 2024-11-09T22:41:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192176895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192176894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192176895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192176895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:57,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9e08ec608309454eac77b7db958d54c7 2024-11-09T22:41:57,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0d9dac3bef8b402ba55b2e1debc7c047 is 50, key is test_row_0/C:col10/1731192115102/Put/seqid=0 2024-11-09T22:41:57,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742294_1470 (size=12301) 2024-11-09T22:41:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T22:41:57,367 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:41:57,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:57,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47010 deadline: 1731192177406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:57,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46934 deadline: 1731192177406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46936 deadline: 1731192177406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:57,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46996 deadline: 1731192177407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:57,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 , the old value is region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:41:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., hostname=9d8c37854f6e,35101,1731191981027, seqNum=2 because the exception is null or not the one we care about 2024-11-09T22:41:57,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0d9dac3bef8b402ba55b2e1debc7c047 2024-11-09T22:41:57,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/d6561e66af31496eb681af64f15909b4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/d6561e66af31496eb681af64f15909b4 2024-11-09T22:41:57,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/d6561e66af31496eb681af64f15909b4, entries=200, sequenceid=352, filesize=14.4 K 2024-11-09T22:41:57,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9e08ec608309454eac77b7db958d54c7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9e08ec608309454eac77b7db958d54c7 2024-11-09T22:41:57,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9e08ec608309454eac77b7db958d54c7, entries=150, sequenceid=352, filesize=12.0 K 2024-11-09T22:41:57,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/0d9dac3bef8b402ba55b2e1debc7c047 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d9dac3bef8b402ba55b2e1debc7c047 2024-11-09T22:41:57,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d9dac3bef8b402ba55b2e1debc7c047, entries=150, sequenceid=352, filesize=12.0 K 2024-11-09T22:41:57,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for bd15f4850dbe30ecfd33655486a40c35 in 1274ms, sequenceid=352, compaction requested=true 2024-11-09T22:41:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:41:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:41:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:57,523 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:57,523 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd15f4850dbe30ecfd33655486a40c35:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:41:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:57,524 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37377 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:57,524 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:57,524 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/B is initiating minor compaction (all files) 2024-11-09T22:41:57,524 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/A is initiating minor compaction (all files) 2024-11-09T22:41:57,524 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/B in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:57,524 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/A in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:57,524 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/cd69e7d93cda4845a6eecb3329d3913e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/301626c179924a00887bf3a5109b924e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9e08ec608309454eac77b7db958d54c7] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=34.3 K 2024-11-09T22:41:57,525 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/cb7061f6385e4e879fdeb4a5492145eb, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8fe8d54d58a4d05a9e23c9300941286, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/d6561e66af31496eb681af64f15909b4] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=36.5 K 2024-11-09T22:41:57,525 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb7061f6385e4e879fdeb4a5492145eb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731192112864 2024-11-09T22:41:57,525 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting cd69e7d93cda4845a6eecb3329d3913e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731192112864 2024-11-09T22:41:57,525 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 301626c179924a00887bf3a5109b924e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731192114081 2024-11-09T22:41:57,525 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8fe8d54d58a4d05a9e23c9300941286, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731192114081 2024-11-09T22:41:57,526 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e08ec608309454eac77b7db958d54c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731192115080 2024-11-09T22:41:57,526 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6561e66af31496eb681af64f15909b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731192115080 2024-11-09T22:41:57,533 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:57,533 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:57,534 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/af54453e37184cf294e789669de5c647 is 50, key is test_row_0/A:col10/1731192115102/Put/seqid=0 2024-11-09T22:41:57,534 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/cbfaadbcf4a642eca47f0c85b35b039c is 50, key is test_row_0/B:col10/1731192115102/Put/seqid=0 2024-11-09T22:41:57,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742295_1471 (size=12881) 2024-11-09T22:41:57,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742296_1472 (size=13017) 2024-11-09T22:41:57,948 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/af54453e37184cf294e789669de5c647 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/af54453e37184cf294e789669de5c647 2024-11-09T22:41:57,948 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/cbfaadbcf4a642eca47f0c85b35b039c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/cbfaadbcf4a642eca47f0c85b35b039c 2024-11-09T22:41:57,953 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/B of bd15f4850dbe30ecfd33655486a40c35 into cbfaadbcf4a642eca47f0c85b35b039c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:57,953 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/A of bd15f4850dbe30ecfd33655486a40c35 into af54453e37184cf294e789669de5c647(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:57,953 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:57,953 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:57,953 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/B, priority=13, startTime=1731192117523; duration=0sec 2024-11-09T22:41:57,953 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/A, priority=13, startTime=1731192117523; duration=0sec 2024-11-09T22:41:57,953 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:57,953 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:B 2024-11-09T22:41:57,953 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:41:57,953 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:A 2024-11-09T22:41:57,953 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:41:57,954 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:41:57,954 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): bd15f4850dbe30ecfd33655486a40c35/C is initiating minor compaction (all files) 2024-11-09T22:41:57,954 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd15f4850dbe30ecfd33655486a40c35/C in TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:57,954 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/f1804b8154cb4d7cb1f38423ef9e50cd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/e5a910d0ba1241fdad19fd32c094f3ae, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d9dac3bef8b402ba55b2e1debc7c047] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp, totalSize=34.1 K 2024-11-09T22:41:57,955 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting f1804b8154cb4d7cb1f38423ef9e50cd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731192112864 2024-11-09T22:41:57,955 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a910d0ba1241fdad19fd32c094f3ae, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731192114081 2024-11-09T22:41:57,955 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d9dac3bef8b402ba55b2e1debc7c047, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731192115080 2024-11-09T22:41:57,962 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd15f4850dbe30ecfd33655486a40c35#C#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:41:57,962 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/68107740a87f4433a391b702a13cb74e is 50, key is test_row_0/C:col10/1731192115102/Put/seqid=0 2024-11-09T22:41:57,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742297_1473 (size=12881) 2024-11-09T22:41:58,379 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/68107740a87f4433a391b702a13cb74e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/68107740a87f4433a391b702a13cb74e 2024-11-09T22:41:58,386 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd15f4850dbe30ecfd33655486a40c35/C of bd15f4850dbe30ecfd33655486a40c35 into 68107740a87f4433a391b702a13cb74e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:41:58,386 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:58,386 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., storeName=bd15f4850dbe30ecfd33655486a40c35/C, priority=13, startTime=1731192117523; duration=0sec 2024-11-09T22:41:58,386 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:41:58,386 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd15f4850dbe30ecfd33655486a40c35:C 2024-11-09T22:41:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:58,419 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:41:58,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:41:58,419 INFO [Thread-1807 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1807. 2024-11-09T22:41:58,419 DEBUG [Thread-1807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:58,419 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:58,420 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:58,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:58,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/36a7c86a572f4a9daedfa4c52bb46e72 is 50, key is test_row_0/A:col10/1731192116261/Put/seqid=0 2024-11-09T22:41:58,427 INFO [Thread-1811 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1811. 2024-11-09T22:41:58,427 INFO [Thread-1803 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1803. 2024-11-09T22:41:58,427 DEBUG [Thread-1803 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:58,427 DEBUG [Thread-1811 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:58,427 INFO [Thread-1805 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1805. 2024-11-09T22:41:58,427 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:58,427 DEBUG [Thread-1811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:58,427 DEBUG [Thread-1805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:58,427 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:58,427 DEBUG [Thread-1811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:58,427 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:58,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:58,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:58,427 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:58,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:58,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742298_1474 (size=12301) 2024-11-09T22:41:58,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/36a7c86a572f4a9daedfa4c52bb46e72 2024-11-09T22:41:58,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9cbb423b42cb4241a094b0c8845f3f3d is 50, key is test_row_0/B:col10/1731192116261/Put/seqid=0 2024-11-09T22:41:58,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742299_1475 (size=12301) 2024-11-09T22:41:59,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9cbb423b42cb4241a094b0c8845f3f3d 2024-11-09T22:41:59,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/25f2601f664b4a629e11b90ffd316527 is 50, key is test_row_0/C:col10/1731192116261/Put/seqid=0 2024-11-09T22:41:59,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742300_1476 (size=12301) 2024-11-09T22:41:59,570 INFO [Thread-1809 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1809. 2024-11-09T22:41:59,570 DEBUG [Thread-1809 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:41:59,570 DEBUG [Thread-1809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:59,571 DEBUG [Thread-1809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:59,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:59,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T22:41:59,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1380 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4140 rows 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1384 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4152 rows 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1379 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4137 rows 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1387 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4155 rows 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1388 2024-11-09T22:41:59,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4158 rows 2024-11-09T22:41:59,572 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:41:59,573 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:41:59,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:59,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:41:59,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:41:59,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T22:41:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T22:41:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T22:41:59,579 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192119579"}]},"ts":"1731192119579"} 2024-11-09T22:41:59,580 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T22:41:59,580 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T22:41:59,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:41:59,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, UNASSIGN}] 2024-11-09T22:41:59,583 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, UNASSIGN 2024-11-09T22:41:59,584 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=bd15f4850dbe30ecfd33655486a40c35, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:41:59,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, UNASSIGN because future has completed 2024-11-09T22:41:59,586 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:41:59,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:41:59,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/25f2601f664b4a629e11b90ffd316527 2024-11-09T22:41:59,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/36a7c86a572f4a9daedfa4c52bb46e72 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/36a7c86a572f4a9daedfa4c52bb46e72 2024-11-09T22:41:59,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/36a7c86a572f4a9daedfa4c52bb46e72, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T22:41:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T22:41:59,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/9cbb423b42cb4241a094b0c8845f3f3d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9cbb423b42cb4241a094b0c8845f3f3d 2024-11-09T22:41:59,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9cbb423b42cb4241a094b0c8845f3f3d, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T22:41:59,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/25f2601f664b4a629e11b90ffd316527 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/25f2601f664b4a629e11b90ffd316527 2024-11-09T22:41:59,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/25f2601f664b4a629e11b90ffd316527, entries=150, sequenceid=366, filesize=12.0 K 2024-11-09T22:41:59,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=26.84 KB/27480 for bd15f4850dbe30ecfd33655486a40c35 in 1276ms, sequenceid=366, compaction requested=false 2024-11-09T22:41:59,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd15f4850dbe30ecfd33655486a40c35: 2024-11-09T22:41:59,738 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:41:59,738 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:41:59,738 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing bd15f4850dbe30ecfd33655486a40c35, disabling compactions & flushes 2024-11-09T22:41:59,739 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:59,739 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:59,739 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. after waiting 0 ms 2024-11-09T22:41:59,739 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:41:59,739 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing bd15f4850dbe30ecfd33655486a40c35 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T22:41:59,745 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/45611e242a004807a2734e385e0f963b is 50, key is test_row_0/A:col10/1731192119566/Put/seqid=0 2024-11-09T22:41:59,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742301_1477 (size=7415) 2024-11-09T22:41:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T22:42:00,152 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/45611e242a004807a2734e385e0f963b 2024-11-09T22:42:00,165 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/d3ed266f29b04b9daf81af73ae62b550 is 50, key is test_row_0/B:col10/1731192119566/Put/seqid=0 2024-11-09T22:42:00,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742302_1478 (size=7415) 2024-11-09T22:42:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T22:42:00,571 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/d3ed266f29b04b9daf81af73ae62b550 2024-11-09T22:42:00,586 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/7d31dd48f4ad4c3ca4c6463f9ec3c5db is 50, key is test_row_0/C:col10/1731192119566/Put/seqid=0 2024-11-09T22:42:00,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742303_1479 (size=7415) 2024-11-09T22:42:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T22:42:00,991 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/7d31dd48f4ad4c3ca4c6463f9ec3c5db 2024-11-09T22:42:01,003 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/A/45611e242a004807a2734e385e0f963b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/45611e242a004807a2734e385e0f963b 2024-11-09T22:42:01,007 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/45611e242a004807a2734e385e0f963b, entries=50, sequenceid=373, filesize=7.2 K 2024-11-09T22:42:01,008 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/B/d3ed266f29b04b9daf81af73ae62b550 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d3ed266f29b04b9daf81af73ae62b550 2024-11-09T22:42:01,012 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d3ed266f29b04b9daf81af73ae62b550, entries=50, sequenceid=373, filesize=7.2 K 2024-11-09T22:42:01,013 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/.tmp/C/7d31dd48f4ad4c3ca4c6463f9ec3c5db as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/7d31dd48f4ad4c3ca4c6463f9ec3c5db 2024-11-09T22:42:01,017 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/7d31dd48f4ad4c3ca4c6463f9ec3c5db, entries=50, sequenceid=373, filesize=7.2 K 2024-11-09T22:42:01,017 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for bd15f4850dbe30ecfd33655486a40c35 in 1278ms, sequenceid=373, compaction requested=true 2024-11-09T22:42:01,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/3796e2dc9a8f459d8bc852994f065473, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/fa66af81c2a64baca044fa60e2990267, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/327a28eee8f6424c86a49c390602fef2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/96ab5588f61346e1bc5c172c7bb006dd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a01516a1f2ac49c693f2c3183b13a74e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b717030cde9e4808aab09f23f1ad8087, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/50022f14da1745cbbe6ef982a3494bdb, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1f077a6710c746e29c4695c2b58acc4e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/0de6d3dc320d4b5db41d2b75a00e15dc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/c919f4368a1d420b926f2f0473461309, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc44416396741e7a99ec377fe30ea18, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1037633a1c714c339c0e7747b9e1cf6c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b8ac68d320a34f20b413e71c22d646bd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/4c0cef9a1d654108bccba4852cbc13f0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/cb7061f6385e4e879fdeb4a5492145eb, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8fe8d54d58a4d05a9e23c9300941286, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/d6561e66af31496eb681af64f15909b4] to archive 2024-11-09T22:42:01,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:42:01,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/3796e2dc9a8f459d8bc852994f065473 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/3796e2dc9a8f459d8bc852994f065473 2024-11-09T22:42:01,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/fa66af81c2a64baca044fa60e2990267 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/fa66af81c2a64baca044fa60e2990267 2024-11-09T22:42:01,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/327a28eee8f6424c86a49c390602fef2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/327a28eee8f6424c86a49c390602fef2 2024-11-09T22:42:01,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/96ab5588f61346e1bc5c172c7bb006dd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/96ab5588f61346e1bc5c172c7bb006dd 2024-11-09T22:42:01,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a01516a1f2ac49c693f2c3183b13a74e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a01516a1f2ac49c693f2c3183b13a74e 2024-11-09T22:42:01,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b717030cde9e4808aab09f23f1ad8087 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b717030cde9e4808aab09f23f1ad8087 2024-11-09T22:42:01,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/50022f14da1745cbbe6ef982a3494bdb to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/50022f14da1745cbbe6ef982a3494bdb 2024-11-09T22:42:01,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1f077a6710c746e29c4695c2b58acc4e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1f077a6710c746e29c4695c2b58acc4e 2024-11-09T22:42:01,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/0de6d3dc320d4b5db41d2b75a00e15dc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/0de6d3dc320d4b5db41d2b75a00e15dc 2024-11-09T22:42:01,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/c919f4368a1d420b926f2f0473461309 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/c919f4368a1d420b926f2f0473461309 2024-11-09T22:42:01,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc44416396741e7a99ec377fe30ea18 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/5bc44416396741e7a99ec377fe30ea18 2024-11-09T22:42:01,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1037633a1c714c339c0e7747b9e1cf6c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/1037633a1c714c339c0e7747b9e1cf6c 2024-11-09T22:42:01,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b8ac68d320a34f20b413e71c22d646bd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/b8ac68d320a34f20b413e71c22d646bd 2024-11-09T22:42:01,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/4c0cef9a1d654108bccba4852cbc13f0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/4c0cef9a1d654108bccba4852cbc13f0 2024-11-09T22:42:01,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/cb7061f6385e4e879fdeb4a5492145eb to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/cb7061f6385e4e879fdeb4a5492145eb 2024-11-09T22:42:01,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8fe8d54d58a4d05a9e23c9300941286 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/a8fe8d54d58a4d05a9e23c9300941286 2024-11-09T22:42:01,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/d6561e66af31496eb681af64f15909b4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/d6561e66af31496eb681af64f15909b4 2024-11-09T22:42:01,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/27c1a2ee967b417596a9686c92801600, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/86d2421bb2094667a025f7b773534699, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/1f14afa785294c3bb12a0d09564285d2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c1adba3b872446c7950d7902c980e48c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/aa4c71bac7bc449aaf93b15e1af2138f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/62b2e24164844ad6bf73f66ef930199a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/6329aa36d4864a48a4a096866fb32430, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9575c0e303f84d98b5da6310e508917d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/ed214fffe8cf4c05945432460b6dd915, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/06d1356043904834b3a7cd8e854d4960, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2dc81729fecd442bbbd44548621e62c4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fbafa7e09dc84e768f23a5501f34dc99, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fc4b5baa4e1e491c819611f90ba8eb02, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/596726ac037845f6be37f6be5c936f4c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/02efd0975362489d8aa11de64df4b9ee, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/bf1408e811d54d4ab78b626c49355e60, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9b0320abe86a43eea677e3758af3198a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/cd69e7d93cda4845a6eecb3329d3913e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c652e90112f14201a45f51e343b65a5c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/301626c179924a00887bf3a5109b924e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9e08ec608309454eac77b7db958d54c7] to archive 2024-11-09T22:42:01,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:42:01,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/27c1a2ee967b417596a9686c92801600 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/27c1a2ee967b417596a9686c92801600 2024-11-09T22:42:01,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/86d2421bb2094667a025f7b773534699 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/86d2421bb2094667a025f7b773534699 2024-11-09T22:42:01,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/1f14afa785294c3bb12a0d09564285d2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/1f14afa785294c3bb12a0d09564285d2 2024-11-09T22:42:01,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c1adba3b872446c7950d7902c980e48c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c1adba3b872446c7950d7902c980e48c 2024-11-09T22:42:01,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/aa4c71bac7bc449aaf93b15e1af2138f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/aa4c71bac7bc449aaf93b15e1af2138f 2024-11-09T22:42:01,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/62b2e24164844ad6bf73f66ef930199a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/62b2e24164844ad6bf73f66ef930199a 2024-11-09T22:42:01,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/6329aa36d4864a48a4a096866fb32430 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/6329aa36d4864a48a4a096866fb32430 2024-11-09T22:42:01,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9575c0e303f84d98b5da6310e508917d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9575c0e303f84d98b5da6310e508917d 2024-11-09T22:42:01,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/ed214fffe8cf4c05945432460b6dd915 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/ed214fffe8cf4c05945432460b6dd915 2024-11-09T22:42:01,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/06d1356043904834b3a7cd8e854d4960 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/06d1356043904834b3a7cd8e854d4960 2024-11-09T22:42:01,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2dc81729fecd442bbbd44548621e62c4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/2dc81729fecd442bbbd44548621e62c4 2024-11-09T22:42:01,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fbafa7e09dc84e768f23a5501f34dc99 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fbafa7e09dc84e768f23a5501f34dc99 2024-11-09T22:42:01,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fc4b5baa4e1e491c819611f90ba8eb02 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/fc4b5baa4e1e491c819611f90ba8eb02 2024-11-09T22:42:01,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/596726ac037845f6be37f6be5c936f4c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/596726ac037845f6be37f6be5c936f4c 2024-11-09T22:42:01,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/02efd0975362489d8aa11de64df4b9ee to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/02efd0975362489d8aa11de64df4b9ee 2024-11-09T22:42:01,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/bf1408e811d54d4ab78b626c49355e60 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/bf1408e811d54d4ab78b626c49355e60 2024-11-09T22:42:01,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9b0320abe86a43eea677e3758af3198a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9b0320abe86a43eea677e3758af3198a 2024-11-09T22:42:01,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/cd69e7d93cda4845a6eecb3329d3913e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/cd69e7d93cda4845a6eecb3329d3913e 2024-11-09T22:42:01,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c652e90112f14201a45f51e343b65a5c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/c652e90112f14201a45f51e343b65a5c 2024-11-09T22:42:01,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/301626c179924a00887bf3a5109b924e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/301626c179924a00887bf3a5109b924e 2024-11-09T22:42:01,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9e08ec608309454eac77b7db958d54c7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9e08ec608309454eac77b7db958d54c7 2024-11-09T22:42:01,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/1d2c26c5f98f46d2b9d94996ec3bc83c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d2c58585403b4b81ae51c30c2426be93, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/213b0c80f1724b4b947e481664ebe961, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/b82862fd903a4034936a406a54c752d9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/861fd11c5cca4af0818868362f9aeccc, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c409c31575214ce69371f33f3230e606, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c5ba6a2c4bf344c093faae753b8bf2a7, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0f10be6ec3c74e6ca310da4bb8077c7c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/14850a92c53647e8ae964e681b920f40, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/832bed9677194ed695a9c06aa6d53cc9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/79440134982743b1ac85c9887120932e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d3085de0ad64609b5e920e3e2347c23, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/bf374ba0de274d20b7a350ab0f7b159f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/f1804b8154cb4d7cb1f38423ef9e50cd, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/9838feb3cd3249b4b2f2d3db88f3f13c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/e5a910d0ba1241fdad19fd32c094f3ae, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d9dac3bef8b402ba55b2e1debc7c047] to archive 2024-11-09T22:42:01,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:42:01,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/1d2c26c5f98f46d2b9d94996ec3bc83c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/1d2c26c5f98f46d2b9d94996ec3bc83c 2024-11-09T22:42:01,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d2c58585403b4b81ae51c30c2426be93 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/d2c58585403b4b81ae51c30c2426be93 2024-11-09T22:42:01,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/213b0c80f1724b4b947e481664ebe961 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/213b0c80f1724b4b947e481664ebe961 2024-11-09T22:42:01,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/b82862fd903a4034936a406a54c752d9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/b82862fd903a4034936a406a54c752d9 2024-11-09T22:42:01,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/861fd11c5cca4af0818868362f9aeccc to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/861fd11c5cca4af0818868362f9aeccc 2024-11-09T22:42:01,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c409c31575214ce69371f33f3230e606 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c409c31575214ce69371f33f3230e606 2024-11-09T22:42:01,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c5ba6a2c4bf344c093faae753b8bf2a7 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/c5ba6a2c4bf344c093faae753b8bf2a7 2024-11-09T22:42:01,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0f10be6ec3c74e6ca310da4bb8077c7c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0f10be6ec3c74e6ca310da4bb8077c7c 2024-11-09T22:42:01,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/14850a92c53647e8ae964e681b920f40 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/14850a92c53647e8ae964e681b920f40 2024-11-09T22:42:01,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/832bed9677194ed695a9c06aa6d53cc9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/832bed9677194ed695a9c06aa6d53cc9 2024-11-09T22:42:01,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/79440134982743b1ac85c9887120932e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/79440134982743b1ac85c9887120932e 2024-11-09T22:42:01,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d3085de0ad64609b5e920e3e2347c23 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d3085de0ad64609b5e920e3e2347c23 2024-11-09T22:42:01,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/bf374ba0de274d20b7a350ab0f7b159f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/bf374ba0de274d20b7a350ab0f7b159f 2024-11-09T22:42:01,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/f1804b8154cb4d7cb1f38423ef9e50cd to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/f1804b8154cb4d7cb1f38423ef9e50cd 2024-11-09T22:42:01,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/9838feb3cd3249b4b2f2d3db88f3f13c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/9838feb3cd3249b4b2f2d3db88f3f13c 2024-11-09T22:42:01,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/e5a910d0ba1241fdad19fd32c094f3ae to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/e5a910d0ba1241fdad19fd32c094f3ae 2024-11-09T22:42:01,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d9dac3bef8b402ba55b2e1debc7c047 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/0d9dac3bef8b402ba55b2e1debc7c047 2024-11-09T22:42:01,072 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-09T22:42:01,073 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. 2024-11-09T22:42:01,073 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for bd15f4850dbe30ecfd33655486a40c35: Waiting for close lock at 1731192119738Running coprocessor pre-close hooks at 1731192119738Disabling compacts and flushes for region at 1731192119738Disabling writes for close at 1731192119739 (+1 ms)Obtaining lock to block concurrent updates at 1731192119739Preparing flush snapshotting stores in bd15f4850dbe30ecfd33655486a40c35 at 1731192119739Finished memstore snapshotting TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731192119739Flushing stores of TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35. at 1731192119740 (+1 ms)Flushing bd15f4850dbe30ecfd33655486a40c35/A: creating writer at 1731192119740Flushing bd15f4850dbe30ecfd33655486a40c35/A: appending metadata at 1731192119744 (+4 ms)Flushing bd15f4850dbe30ecfd33655486a40c35/A: closing flushed file at 1731192119744Flushing bd15f4850dbe30ecfd33655486a40c35/B: creating writer at 1731192120161 (+417 ms)Flushing bd15f4850dbe30ecfd33655486a40c35/B: appending metadata at 1731192120164 (+3 ms)Flushing bd15f4850dbe30ecfd33655486a40c35/B: closing flushed file at 1731192120164Flushing bd15f4850dbe30ecfd33655486a40c35/C: creating writer at 1731192120580 (+416 ms)Flushing bd15f4850dbe30ecfd33655486a40c35/C: appending metadata at 1731192120585 (+5 ms)Flushing bd15f4850dbe30ecfd33655486a40c35/C: closing flushed file at 1731192120585Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d6313dc: reopening flushed file at 1731192121002 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a464b58: reopening flushed file at 1731192121007 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@797514a7: reopening flushed file at 1731192121012 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for bd15f4850dbe30ecfd33655486a40c35 in 1278ms, sequenceid=373, compaction requested=true at 1731192121017 (+5 ms)Writing region close event to WAL at 1731192121070 (+53 ms)Running coprocessor post-close hooks at 1731192121073 (+3 ms)Closed at 1731192121073 2024-11-09T22:42:01,075 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:42:01,075 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=bd15f4850dbe30ecfd33655486a40c35, regionState=CLOSED 2024-11-09T22:42:01,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:42:01,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-09T22:42:01,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure bd15f4850dbe30ecfd33655486a40c35, server=9d8c37854f6e,35101,1731191981027 in 1.4910 sec 2024-11-09T22:42:01,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-09T22:42:01,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd15f4850dbe30ecfd33655486a40c35, UNASSIGN in 1.4950 sec 2024-11-09T22:42:01,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-09T22:42:01,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4990 sec 2024-11-09T22:42:01,083 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192121082"}]},"ts":"1731192121082"} 2024-11-09T22:42:01,084 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T22:42:01,084 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T22:42:01,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-09T22:42:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T22:42:01,727 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:01,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T22:42:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:01,733 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T22:42:01,735 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:01,740 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:42:01,744 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/recovered.edits] 2024-11-09T22:42:01,750 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/36a7c86a572f4a9daedfa4c52bb46e72 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/36a7c86a572f4a9daedfa4c52bb46e72 2024-11-09T22:42:01,753 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/45611e242a004807a2734e385e0f963b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/45611e242a004807a2734e385e0f963b 2024-11-09T22:42:01,755 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/af54453e37184cf294e789669de5c647 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/A/af54453e37184cf294e789669de5c647 2024-11-09T22:42:01,760 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9cbb423b42cb4241a094b0c8845f3f3d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/9cbb423b42cb4241a094b0c8845f3f3d 2024-11-09T22:42:01,763 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/cbfaadbcf4a642eca47f0c85b35b039c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/cbfaadbcf4a642eca47f0c85b35b039c 2024-11-09T22:42:01,766 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d3ed266f29b04b9daf81af73ae62b550 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/B/d3ed266f29b04b9daf81af73ae62b550 2024-11-09T22:42:01,770 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/25f2601f664b4a629e11b90ffd316527 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/25f2601f664b4a629e11b90ffd316527 2024-11-09T22:42:01,773 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/68107740a87f4433a391b702a13cb74e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/68107740a87f4433a391b702a13cb74e 2024-11-09T22:42:01,774 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/7d31dd48f4ad4c3ca4c6463f9ec3c5db to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/C/7d31dd48f4ad4c3ca4c6463f9ec3c5db 2024-11-09T22:42:01,779 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/recovered.edits/376.seqid to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35/recovered.edits/376.seqid 2024-11-09T22:42:01,780 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/bd15f4850dbe30ecfd33655486a40c35 2024-11-09T22:42:01,780 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T22:42:01,784 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:01,787 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T22:42:01,790 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T22:42:01,791 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:01,792 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T22:42:01,792 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731192121792"}]},"ts":"9223372036854775807"} 2024-11-09T22:42:01,794 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T22:42:01,795 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bd15f4850dbe30ecfd33655486a40c35, NAME => 'TestAcidGuarantees,,1731192094164.bd15f4850dbe30ecfd33655486a40c35.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T22:42:01,795 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T22:42:01,795 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731192121795"}]},"ts":"9223372036854775807"} 2024-11-09T22:42:01,797 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T22:42:01,797 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:01,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-09T22:42:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T22:42:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T22:42:01,852 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:01,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 240), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=17621 (was 17637) 2024-11-09T22:42:01,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=17620 2024-11-09T22:42:01,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:42:01,873 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:42:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:01,875 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T22:42:01,875 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:01,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-09T22:42:01,876 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T22:42:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T22:42:01,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742304_1480 (size=959) 2024-11-09T22:42:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T22:42:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T22:42:02,286 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696 2024-11-09T22:42:02,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742305_1481 (size=53) 2024-11-09T22:42:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T22:42:02,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:42:02,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0af038d6aa643579358dbb0f202b0f57, disabling compactions & flushes 2024-11-09T22:42:02,697 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:02,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:02,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. after waiting 0 ms 2024-11-09T22:42:02,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:02,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:02,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0af038d6aa643579358dbb0f202b0f57: Waiting for close lock at 1731192122697Disabling compacts and flushes for region at 1731192122697Disabling writes for close at 1731192122698 (+1 ms)Writing region close event to WAL at 1731192122698Closed at 1731192122698 2024-11-09T22:42:02,701 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T22:42:02,702 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731192122701"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731192122701"}]},"ts":"1731192122701"} 2024-11-09T22:42:02,704 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T22:42:02,705 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T22:42:02,705 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192122705"}]},"ts":"1731192122705"} 2024-11-09T22:42:02,707 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T22:42:02,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, ASSIGN}] 2024-11-09T22:42:02,709 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, ASSIGN 2024-11-09T22:42:02,710 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, ASSIGN; state=OFFLINE, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=false 2024-11-09T22:42:02,861 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0af038d6aa643579358dbb0f202b0f57, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:02,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, ASSIGN because future has completed 2024-11-09T22:42:02,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:42:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T22:42:03,025 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:03,025 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:42:03,026 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,026 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:42:03,026 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,027 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,029 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,031 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af038d6aa643579358dbb0f202b0f57 columnFamilyName A 2024-11-09T22:42:03,031 DEBUG [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:03,031 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(327): Store=0af038d6aa643579358dbb0f202b0f57/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:42:03,032 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,033 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af038d6aa643579358dbb0f202b0f57 columnFamilyName B 2024-11-09T22:42:03,033 DEBUG [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:03,033 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(327): Store=0af038d6aa643579358dbb0f202b0f57/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:42:03,034 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,035 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af038d6aa643579358dbb0f202b0f57 columnFamilyName C 2024-11-09T22:42:03,035 DEBUG [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:03,035 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(327): Store=0af038d6aa643579358dbb0f202b0f57/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:42:03,036 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,036 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,037 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,038 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,038 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,039 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:42:03,041 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,043 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T22:42:03,044 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 0af038d6aa643579358dbb0f202b0f57; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72365548, jitterRate=0.07833069562911987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:42:03,044 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:03,045 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 0af038d6aa643579358dbb0f202b0f57: Running coprocessor pre-open hook at 1731192123027Writing region info on filesystem at 1731192123027Initializing all the Stores at 1731192123029 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192123029Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192123029Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192123029Cleaning up temporary data from old regions at 1731192123038 (+9 ms)Running coprocessor post-open hooks at 1731192123044 (+6 ms)Region opened successfully at 1731192123045 (+1 ms) 2024-11-09T22:42:03,046 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., pid=140, masterSystemTime=1731192123018 2024-11-09T22:42:03,047 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:03,048 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:03,048 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0af038d6aa643579358dbb0f202b0f57, regionState=OPEN, openSeqNum=2, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:03,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:42:03,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-09T22:42:03,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 in 186 msec 2024-11-09T22:42:03,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-09T22:42:03,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, ASSIGN in 344 msec 2024-11-09T22:42:03,054 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T22:42:03,054 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192123054"}]},"ts":"1731192123054"} 2024-11-09T22:42:03,055 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T22:42:03,056 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T22:42:03,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-09T22:42:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-09T22:42:04,027 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:04,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:04,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:04,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:04,032 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d16815, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:04,037 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a700b57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:04,038 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:04,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:04,042 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T22:42:04,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d8c37854f6e,34651,1731191980160 2024-11-09T22:42:04,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@642f6921 2024-11-09T22:42:04,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T22:42:04,043 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T22:42:04,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T22:42:04,045 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T22:42:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:04,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742306_1482 (size=995) 2024-11-09T22:42:04,459 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-09T22:42:04,459 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-09T22:42:04,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:42:04,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, REOPEN/MOVE}] 2024-11-09T22:42:04,469 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, REOPEN/MOVE 2024-11-09T22:42:04,470 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0af038d6aa643579358dbb0f202b0f57, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:04,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, REOPEN/MOVE because future has completed 2024-11-09T22:42:04,471 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:42:04,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:42:04,624 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,625 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:42:04,625 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 0af038d6aa643579358dbb0f202b0f57, disabling compactions & flushes 2024-11-09T22:42:04,625 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:04,625 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:04,625 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. after waiting 0 ms 2024-11-09T22:42:04,625 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:04,633 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T22:42:04,634 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:04,634 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 0af038d6aa643579358dbb0f202b0f57: Waiting for close lock at 1731192124625Running coprocessor pre-close hooks at 1731192124625Disabling compacts and flushes for region at 1731192124625Disabling writes for close at 1731192124625Writing region close event to WAL at 1731192124626 (+1 ms)Running coprocessor post-close hooks at 1731192124634 (+8 ms)Closed at 1731192124634 2024-11-09T22:42:04,634 WARN [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 0af038d6aa643579358dbb0f202b0f57 to self. 2024-11-09T22:42:04,638 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,638 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0af038d6aa643579358dbb0f202b0f57, regionState=CLOSED 2024-11-09T22:42:04,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:42:04,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-09T22:42:04,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 in 170 msec 2024-11-09T22:42:04,644 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, REOPEN/MOVE; state=CLOSED, location=9d8c37854f6e,35101,1731191981027; forceNewPlan=false, retain=true 2024-11-09T22:42:04,795 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0af038d6aa643579358dbb0f202b0f57, regionState=OPENING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:04,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, REOPEN/MOVE because future has completed 2024-11-09T22:42:04,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:42:04,960 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:04,960 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} 2024-11-09T22:42:04,961 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,961 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T22:42:04,961 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,961 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,963 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,964 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af038d6aa643579358dbb0f202b0f57 columnFamilyName A 2024-11-09T22:42:04,966 DEBUG [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:04,966 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(327): Store=0af038d6aa643579358dbb0f202b0f57/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:42:04,967 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,968 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af038d6aa643579358dbb0f202b0f57 columnFamilyName B 2024-11-09T22:42:04,968 DEBUG [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:04,969 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(327): Store=0af038d6aa643579358dbb0f202b0f57/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:42:04,969 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,970 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0af038d6aa643579358dbb0f202b0f57 columnFamilyName C 2024-11-09T22:42:04,970 DEBUG [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:04,970 INFO [StoreOpener-0af038d6aa643579358dbb0f202b0f57-1 {}] regionserver.HStore(327): Store=0af038d6aa643579358dbb0f202b0f57/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T22:42:04,970 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,972 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,973 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,975 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,975 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,976 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T22:42:04,978 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,980 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 0af038d6aa643579358dbb0f202b0f57; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65982461, jitterRate=-0.01678471267223358}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T22:42:04,980 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:04,981 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 0af038d6aa643579358dbb0f202b0f57: Running coprocessor pre-open hook at 1731192124961Writing region info on filesystem at 1731192124961Initializing all the Stores at 1731192124962 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192124963 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192124963Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731192124963Cleaning up temporary data from old regions at 1731192124975 (+12 ms)Running coprocessor post-open hooks at 1731192124980 (+5 ms)Region opened successfully at 1731192124981 (+1 ms) 2024-11-09T22:42:04,983 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., pid=145, masterSystemTime=1731192124954 2024-11-09T22:42:04,985 DEBUG [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:04,986 INFO [RS_OPEN_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:04,986 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0af038d6aa643579358dbb0f202b0f57, regionState=OPEN, openSeqNum=5, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:04,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:42:04,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-09T22:42:04,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 in 188 msec 2024-11-09T22:42:04,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-09T22:42:04,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, REOPEN/MOVE in 521 msec 2024-11-09T22:42:04,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-09T22:42:04,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-11-09T22:42:04,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 949 msec 2024-11-09T22:42:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-09T22:42:04,996 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:04,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:04,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:04,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:04,998 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e5639e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:04,999 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca8bd81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,000 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb34e5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794597f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740c2d76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d303fe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428a95fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc8ab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,007 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f6830e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,008 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77245932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,010 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65dac45f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,011 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12663782, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,012 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6310a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,013 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a538213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,015 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d94fcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,016 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd905d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13889207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e25f42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d8c37854f6e,34651,-1 for getting cluster id 2024-11-09T22:42:05,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T22:42:05,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c19612e1-c4b9-4038-b012-a363c57ebb38' 2024-11-09T22:42:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T22:42:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c19612e1-c4b9-4038-b012-a363c57ebb38" 2024-11-09T22:42:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7310c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d8c37854f6e,34651,-1] 2024-11-09T22:42:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T22:42:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:05,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T22:42:05,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284f2ad4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T22:42:05,022 DEBUG [Thread-2167 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,022 DEBUG [Thread-2175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,022 DEBUG [Thread-2173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,022 DEBUG [Thread-2186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:42:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,023 DEBUG [Thread-2169 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,023 DEBUG [Thread-2173 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,023 DEBUG [Thread-2173 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,024 DEBUG [Thread-2171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,024 DEBUG [Thread-2182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-09T22:42:05,026 DEBUG [Thread-2180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,026 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:42:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T22:42:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,028 DEBUG [Thread-2178 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,029 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:42:05,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:42:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:05,033 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:42:05,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:42:05,034 DEBUG [Thread-2184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T22:42:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d8c37854f6e,35101,1731191981027, seqNum=-1] 2024-11-09T22:42:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T22:42:05,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T22:42:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5] 2024-11-09T22:42:05,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192185057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192185055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192185059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192185061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192185062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110924e42b4accd34974bfa5973a282c8046_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192125033/Put/seqid=0 2024-11-09T22:42:05,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742307_1483 (size=12154) 2024-11-09T22:42:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T22:42:05,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192185165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192185165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192185165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192185175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192185175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T22:42:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:05,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:05,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:05,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T22:42:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:05,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T22:42:05,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192185375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192185375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192185375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192185385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192185385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,476 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:05,479 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110924e42b4accd34974bfa5973a282c8046_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110924e42b4accd34974bfa5973a282c8046_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:05,480 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/07faa4a7068243c29e9ba0b670d483ac, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:05,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/07faa4a7068243c29e9ba0b670d483ac is 175, key is test_row_0/A:col10/1731192125033/Put/seqid=0 2024-11-09T22:42:05,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742308_1484 (size=30955) 2024-11-09T22:42:05,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/07faa4a7068243c29e9ba0b670d483ac 2024-11-09T22:42:05,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T22:42:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:05,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:05,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/47822e5f8c9f404b8829bae9680ec5c8 is 50, key is test_row_0/B:col10/1731192125033/Put/seqid=0 2024-11-09T22:42:05,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742309_1485 (size=12001) 2024-11-09T22:42:05,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/47822e5f8c9f404b8829bae9680ec5c8 2024-11-09T22:42:05,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/4f484f5faafa4ad3bb3ce90a34cb92fa is 50, key is test_row_0/C:col10/1731192125033/Put/seqid=0 2024-11-09T22:42:05,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742310_1486 (size=12001) 2024-11-09T22:42:05,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/4f484f5faafa4ad3bb3ce90a34cb92fa 2024-11-09T22:42:05,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/07faa4a7068243c29e9ba0b670d483ac as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/07faa4a7068243c29e9ba0b670d483ac 2024-11-09T22:42:05,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/07faa4a7068243c29e9ba0b670d483ac, entries=150, sequenceid=17, filesize=30.2 K 2024-11-09T22:42:05,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/47822e5f8c9f404b8829bae9680ec5c8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/47822e5f8c9f404b8829bae9680ec5c8 2024-11-09T22:42:05,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/47822e5f8c9f404b8829bae9680ec5c8, entries=150, sequenceid=17, filesize=11.7 K 2024-11-09T22:42:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/4f484f5faafa4ad3bb3ce90a34cb92fa as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/4f484f5faafa4ad3bb3ce90a34cb92fa 2024-11-09T22:42:05,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/4f484f5faafa4ad3bb3ce90a34cb92fa, entries=150, sequenceid=17, filesize=11.7 K 2024-11-09T22:42:05,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0af038d6aa643579358dbb0f202b0f57 in 514ms, sequenceid=17, compaction requested=false 2024-11-09T22:42:05,547 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T22:42:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:05,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-09T22:42:05,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:05,641 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T22:42:05,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091879a547e1ca4dd7810821cd4ea8c57c_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192125059/Put/seqid=0 2024-11-09T22:42:05,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742311_1487 (size=12154) 2024-11-09T22:42:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T22:42:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:05,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:05,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192185693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192185695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192185695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192185696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192185697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192185805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192185805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192185805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192186015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192186015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192186015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:06,054 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091879a547e1ca4dd7810821cd4ea8c57c_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091879a547e1ca4dd7810821cd4ea8c57c_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:06,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/38ea7e326d1d4ef8a5d1e50845ef4773, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:06,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/38ea7e326d1d4ef8a5d1e50845ef4773 is 175, key is test_row_0/A:col10/1731192125059/Put/seqid=0 2024-11-09T22:42:06,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742312_1488 (size=30955) 2024-11-09T22:42:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T22:42:06,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192186204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192186205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192186324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192186324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192186325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,459 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/38ea7e326d1d4ef8a5d1e50845ef4773 2024-11-09T22:42:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/96013d2d490d4f4298a732418e510f4a is 50, key is test_row_0/B:col10/1731192125059/Put/seqid=0 2024-11-09T22:42:06,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742313_1489 (size=12001) 2024-11-09T22:42:06,589 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T22:42:06,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192186835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192186835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192186835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:06,881 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/96013d2d490d4f4298a732418e510f4a 2024-11-09T22:42:06,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/5266b8b1fe4e4e2ab3f44f21d687d8c9 is 50, key is test_row_0/C:col10/1731192125059/Put/seqid=0 2024-11-09T22:42:06,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742314_1490 (size=12001) 2024-11-09T22:42:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T22:42:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192187215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:07,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192187215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:07,218 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 0af038d6aa643579358dbb0f202b0f57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:42:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:07,218 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 0af038d6aa643579358dbb0f202b0f57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:42:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:07,290 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/5266b8b1fe4e4e2ab3f44f21d687d8c9 2024-11-09T22:42:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/38ea7e326d1d4ef8a5d1e50845ef4773 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/38ea7e326d1d4ef8a5d1e50845ef4773 2024-11-09T22:42:07,343 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/38ea7e326d1d4ef8a5d1e50845ef4773, entries=150, sequenceid=41, filesize=30.2 K 2024-11-09T22:42:07,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/96013d2d490d4f4298a732418e510f4a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96013d2d490d4f4298a732418e510f4a 2024-11-09T22:42:07,346 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96013d2d490d4f4298a732418e510f4a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T22:42:07,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/5266b8b1fe4e4e2ab3f44f21d687d8c9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/5266b8b1fe4e4e2ab3f44f21d687d8c9 2024-11-09T22:42:07,350 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/5266b8b1fe4e4e2ab3f44f21d687d8c9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T22:42:07,351 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0af038d6aa643579358dbb0f202b0f57 in 1710ms, sequenceid=41, compaction requested=false 2024-11-09T22:42:07,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:07,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:07,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-09T22:42:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-09T22:42:07,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-09T22:42:07,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-11-09T22:42:07,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.3330 sec 2024-11-09T22:42:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:07,874 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:42:07,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T22:42:07,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b8f12372eb2b4271acc9d03ffdf97604_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192127861/Put/seqid=0 2024-11-09T22:42:07,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742315_1491 (size=12154) 2024-11-09T22:42:07,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:07,891 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b8f12372eb2b4271acc9d03ffdf97604_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b8f12372eb2b4271acc9d03ffdf97604_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:07,891 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/81a519f34c504a4682687c56d4220d14, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:07,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/81a519f34c504a4682687c56d4220d14 is 175, key is test_row_0/A:col10/1731192127861/Put/seqid=0 2024-11-09T22:42:07,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742316_1492 (size=30955) 2024-11-09T22:42:07,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192187904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:07,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192187905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192187908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192188015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192188015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192188015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192188225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192188225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192188225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,295 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/81a519f34c504a4682687c56d4220d14 2024-11-09T22:42:08,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/c0fda3840c8b40319f361b9d60d0c50b is 50, key is test_row_0/B:col10/1731192127861/Put/seqid=0 2024-11-09T22:42:08,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742317_1493 (size=12001) 2024-11-09T22:42:08,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/c0fda3840c8b40319f361b9d60d0c50b 2024-11-09T22:42:08,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/9ce4cf99e28b4a56bb11f83b93d90512 is 50, key is test_row_0/C:col10/1731192127861/Put/seqid=0 2024-11-09T22:42:08,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742318_1494 (size=12001) 2024-11-09T22:42:08,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192188535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192188535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192188535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:08,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:08,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/9ce4cf99e28b4a56bb11f83b93d90512 2024-11-09T22:42:08,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/81a519f34c504a4682687c56d4220d14 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/81a519f34c504a4682687c56d4220d14 2024-11-09T22:42:08,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/81a519f34c504a4682687c56d4220d14, entries=150, sequenceid=56, filesize=30.2 K 2024-11-09T22:42:08,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/c0fda3840c8b40319f361b9d60d0c50b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c0fda3840c8b40319f361b9d60d0c50b 2024-11-09T22:42:08,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c0fda3840c8b40319f361b9d60d0c50b, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T22:42:08,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/9ce4cf99e28b4a56bb11f83b93d90512 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/9ce4cf99e28b4a56bb11f83b93d90512 2024-11-09T22:42:08,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/9ce4cf99e28b4a56bb11f83b93d90512, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T22:42:08,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0af038d6aa643579358dbb0f202b0f57 in 855ms, sequenceid=56, compaction requested=true 2024-11-09T22:42:08,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:08,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:42:08,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:08,729 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:42:08,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:42:08,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:08,729 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:42:08,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:42:08,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:08,730 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:42:08,730 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/B is initiating minor compaction (all files) 2024-11-09T22:42:08,730 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:42:08,730 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/B in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:08,730 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/A is initiating minor compaction (all files) 2024-11-09T22:42:08,730 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/A in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:08,730 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/47822e5f8c9f404b8829bae9680ec5c8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96013d2d490d4f4298a732418e510f4a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c0fda3840c8b40319f361b9d60d0c50b] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=35.2 K 2024-11-09T22:42:08,730 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/07faa4a7068243c29e9ba0b670d483ac, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/38ea7e326d1d4ef8a5d1e50845ef4773, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/81a519f34c504a4682687c56d4220d14] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=90.7 K 2024-11-09T22:42:08,730 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:08,730 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/07faa4a7068243c29e9ba0b670d483ac, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/38ea7e326d1d4ef8a5d1e50845ef4773, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/81a519f34c504a4682687c56d4220d14] 2024-11-09T22:42:08,730 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 47822e5f8c9f404b8829bae9680ec5c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731192125032 2024-11-09T22:42:08,730 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07faa4a7068243c29e9ba0b670d483ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731192125032 2024-11-09T22:42:08,731 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 96013d2d490d4f4298a732418e510f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731192125055 2024-11-09T22:42:08,731 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ea7e326d1d4ef8a5d1e50845ef4773, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731192125055 2024-11-09T22:42:08,731 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c0fda3840c8b40319f361b9d60d0c50b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731192127845 2024-11-09T22:42:08,731 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81a519f34c504a4682687c56d4220d14, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731192127845 2024-11-09T22:42:08,738 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:08,742 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#B#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:08,742 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/bcfec35b8ce04835aedcdca6c56d5f1c is 50, key is test_row_0/B:col10/1731192127861/Put/seqid=0 2024-11-09T22:42:08,747 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110941729452a7e543f38ea8220966316b8d_0af038d6aa643579358dbb0f202b0f57 store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:08,749 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110941729452a7e543f38ea8220966316b8d_0af038d6aa643579358dbb0f202b0f57, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:08,749 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110941729452a7e543f38ea8220966316b8d_0af038d6aa643579358dbb0f202b0f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:08,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742319_1495 (size=12104) 2024-11-09T22:42:08,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742320_1496 (size=4469) 2024-11-09T22:42:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:09,047 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:42:09,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T22:42:09,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b55ee2ccccd841289de13aac87fe410b_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192129047/Put/seqid=0 2024-11-09T22:42:09,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742321_1497 (size=14594) 2024-11-09T22:42:09,061 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:09,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b55ee2ccccd841289de13aac87fe410b_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b55ee2ccccd841289de13aac87fe410b_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:09,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/347c6a2812704108ae424b792c3ad331, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:09,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/347c6a2812704108ae424b792c3ad331 is 175, key is test_row_0/A:col10/1731192129047/Put/seqid=0 2024-11-09T22:42:09,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742322_1498 (size=39549) 2024-11-09T22:42:09,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192189065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192189066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192189067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,156 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/bcfec35b8ce04835aedcdca6c56d5f1c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/bcfec35b8ce04835aedcdca6c56d5f1c 2024-11-09T22:42:09,160 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/B of 0af038d6aa643579358dbb0f202b0f57 into bcfec35b8ce04835aedcdca6c56d5f1c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:09,160 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:09,160 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/B, priority=13, startTime=1731192128729; duration=0sec 2024-11-09T22:42:09,160 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:09,160 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:B 2024-11-09T22:42:09,160 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#A#compaction#422 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:09,160 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:42:09,161 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/b14239b40d8647ca96e2889f818275c4 is 175, key is test_row_0/A:col10/1731192127861/Put/seqid=0 2024-11-09T22:42:09,161 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:42:09,161 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/C is initiating minor compaction (all files) 2024-11-09T22:42:09,161 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/C in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,161 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/4f484f5faafa4ad3bb3ce90a34cb92fa, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/5266b8b1fe4e4e2ab3f44f21d687d8c9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/9ce4cf99e28b4a56bb11f83b93d90512] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=35.2 K 2024-11-09T22:42:09,161 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f484f5faafa4ad3bb3ce90a34cb92fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731192125032 2024-11-09T22:42:09,162 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5266b8b1fe4e4e2ab3f44f21d687d8c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731192125055 2024-11-09T22:42:09,162 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ce4cf99e28b4a56bb11f83b93d90512, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731192127845 2024-11-09T22:42:09,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742323_1499 (size=31058) 2024-11-09T22:42:09,169 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/b14239b40d8647ca96e2889f818275c4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/b14239b40d8647ca96e2889f818275c4 2024-11-09T22:42:09,169 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:09,170 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/24ffce2873ba49ee8e7db09620a55a43 is 50, key is test_row_0/C:col10/1731192127861/Put/seqid=0 2024-11-09T22:42:09,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742324_1500 (size=12104) 2024-11-09T22:42:09,174 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/A of 0af038d6aa643579358dbb0f202b0f57 into b14239b40d8647ca96e2889f818275c4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:09,174 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:09,174 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/A, priority=13, startTime=1731192128729; duration=0sec 2024-11-09T22:42:09,174 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:09,174 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:A 2024-11-09T22:42:09,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192189175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192189175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192189175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-09T22:42:09,185 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:09,186 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:42:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-09T22:42:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T22:42:09,187 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:42:09,188 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:42:09,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:42:09,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192189224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,227 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 0af038d6aa643579358dbb0f202b0f57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:42:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192189235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,237 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 0af038d6aa643579358dbb0f202b0f57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:42:09,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T22:42:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T22:42:09,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T22:42:09,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192189385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192189385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192189385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/347c6a2812704108ae424b792c3ad331 2024-11-09T22:42:09,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/96bb8790d4874382b6c5db27d9d1f749 is 50, key is test_row_0/B:col10/1731192129047/Put/seqid=0 2024-11-09T22:42:09,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742325_1501 (size=12001) 2024-11-09T22:42:09,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T22:42:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T22:42:09,578 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/24ffce2873ba49ee8e7db09620a55a43 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/24ffce2873ba49ee8e7db09620a55a43 2024-11-09T22:42:09,581 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/C of 0af038d6aa643579358dbb0f202b0f57 into 24ffce2873ba49ee8e7db09620a55a43(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:09,581 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:09,581 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/C, priority=13, startTime=1731192128729; duration=0sec 2024-11-09T22:42:09,582 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:09,582 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:C 2024-11-09T22:42:09,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T22:42:09,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192189695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192189695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192189695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:09,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T22:42:09,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:09,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T22:42:09,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/96bb8790d4874382b6c5db27d9d1f749 2024-11-09T22:42:09,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/29243fdd264b4ff49cc0189bf650a5ea is 50, key is test_row_0/C:col10/1731192129047/Put/seqid=0 2024-11-09T22:42:09,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742326_1502 (size=12001) 2024-11-09T22:42:09,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T22:42:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:09,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:10,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T22:42:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:10,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:10,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192190204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192190205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192190205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:10,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T22:42:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:10,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:10,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/29243fdd264b4ff49cc0189bf650a5ea 2024-11-09T22:42:10,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/347c6a2812704108ae424b792c3ad331 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/347c6a2812704108ae424b792c3ad331 2024-11-09T22:42:10,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/347c6a2812704108ae424b792c3ad331, entries=200, sequenceid=80, filesize=38.6 K 2024-11-09T22:42:10,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/96bb8790d4874382b6c5db27d9d1f749 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96bb8790d4874382b6c5db27d9d1f749 2024-11-09T22:42:10,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96bb8790d4874382b6c5db27d9d1f749, entries=150, sequenceid=80, filesize=11.7 K 2024-11-09T22:42:10,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/29243fdd264b4ff49cc0189bf650a5ea as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/29243fdd264b4ff49cc0189bf650a5ea 2024-11-09T22:42:10,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/29243fdd264b4ff49cc0189bf650a5ea, entries=150, sequenceid=80, filesize=11.7 K 2024-11-09T22:42:10,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0af038d6aa643579358dbb0f202b0f57 in 1257ms, sequenceid=80, compaction requested=false 2024-11-09T22:42:10,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T22:42:10,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-09T22:42:10,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:10,408 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:42:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109bf48bff627e54993b6b2f1ab6498947e_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192129051/Put/seqid=0 2024-11-09T22:42:10,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742327_1503 (size=12154) 2024-11-09T22:42:10,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:10,819 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109bf48bff627e54993b6b2f1ab6498947e_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109bf48bff627e54993b6b2f1ab6498947e_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:10,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/320011209020494395f26dd5061f4968, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:10,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/320011209020494395f26dd5061f4968 is 175, key is test_row_0/A:col10/1731192129051/Put/seqid=0 2024-11-09T22:42:10,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742328_1504 (size=30955) 2024-11-09T22:42:10,824 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/320011209020494395f26dd5061f4968 2024-11-09T22:42:10,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/4d4f8d76e15d464780b84538b1fdf4bb is 50, key is test_row_0/B:col10/1731192129051/Put/seqid=0 2024-11-09T22:42:10,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742329_1505 (size=12001) 2024-11-09T22:42:11,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:11,233 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/4d4f8d76e15d464780b84538b1fdf4bb 2024-11-09T22:42:11,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/a9c8dbf74307484db2a0d19500f8dbee is 50, key is test_row_0/C:col10/1731192129051/Put/seqid=0 2024-11-09T22:42:11,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742330_1506 (size=12001) 2024-11-09T22:42:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192191250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192191251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192191251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T22:42:11,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192191365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192191365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192191365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192191575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192191575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192191575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,642 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/a9c8dbf74307484db2a0d19500f8dbee 2024-11-09T22:42:11,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/320011209020494395f26dd5061f4968 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/320011209020494395f26dd5061f4968 2024-11-09T22:42:11,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/320011209020494395f26dd5061f4968, entries=150, sequenceid=95, filesize=30.2 K 2024-11-09T22:42:11,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/4d4f8d76e15d464780b84538b1fdf4bb as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4d4f8d76e15d464780b84538b1fdf4bb 2024-11-09T22:42:11,652 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4d4f8d76e15d464780b84538b1fdf4bb, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T22:42:11,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/a9c8dbf74307484db2a0d19500f8dbee as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/a9c8dbf74307484db2a0d19500f8dbee 2024-11-09T22:42:11,655 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/a9c8dbf74307484db2a0d19500f8dbee, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T22:42:11,656 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0af038d6aa643579358dbb0f202b0f57 in 1249ms, sequenceid=95, compaction requested=true 2024-11-09T22:42:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-09T22:42:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-09T22:42:11,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-09T22:42:11,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4690 sec 2024-11-09T22:42:11,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.4730 sec 2024-11-09T22:42:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:11,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:42:11,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:42:11,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110935595c32fa7d413eabff2d32ccb89140_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192131897/Put/seqid=0 2024-11-09T22:42:11,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192191905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192191907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192191908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:11,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742331_1507 (size=17034) 2024-11-09T22:42:12,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192192015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192192015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192192015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192192224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192192225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192192225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,338 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:12,341 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110935595c32fa7d413eabff2d32ccb89140_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110935595c32fa7d413eabff2d32ccb89140_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:12,341 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/a4a59ee4e39c48f3abeb6f2f3818dfb4, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:12,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/a4a59ee4e39c48f3abeb6f2f3818dfb4 is 175, key is test_row_0/A:col10/1731192131897/Put/seqid=0 2024-11-09T22:42:12,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742332_1508 (size=48139) 2024-11-09T22:42:12,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192192535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192192535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192192535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:12,745 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/a4a59ee4e39c48f3abeb6f2f3818dfb4 2024-11-09T22:42:12,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/4a4295d5b0b94708b5c51fa0e1b93c2a is 50, key is test_row_0/B:col10/1731192131897/Put/seqid=0 2024-11-09T22:42:12,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742333_1509 (size=12001) 2024-11-09T22:42:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192193045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192193045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192193045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:13,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/4a4295d5b0b94708b5c51fa0e1b93c2a 2024-11-09T22:42:13,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/2be8a02f353f479984d8e2378c25ab95 is 50, key is test_row_0/C:col10/1731192131897/Put/seqid=0 2024-11-09T22:42:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742334_1510 (size=12001) 2024-11-09T22:42:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192193244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:13,248 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 0af038d6aa643579358dbb0f202b0f57 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:42:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:13,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192193255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:13,258 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 0af038d6aa643579358dbb0f202b0f57 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:42:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-09T22:42:13,345 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:13,345 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:42:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-09T22:42:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T22:42:13,346 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:42:13,347 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:42:13,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:42:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T22:42:13,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T22:42:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:13,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:13,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/2be8a02f353f479984d8e2378c25ab95 2024-11-09T22:42:13,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/a4a59ee4e39c48f3abeb6f2f3818dfb4 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/a4a59ee4e39c48f3abeb6f2f3818dfb4 2024-11-09T22:42:13,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/a4a59ee4e39c48f3abeb6f2f3818dfb4, entries=250, sequenceid=122, filesize=47.0 K 2024-11-09T22:42:13,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/4a4295d5b0b94708b5c51fa0e1b93c2a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4a4295d5b0b94708b5c51fa0e1b93c2a 2024-11-09T22:42:13,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4a4295d5b0b94708b5c51fa0e1b93c2a, entries=150, sequenceid=122, filesize=11.7 K 2024-11-09T22:42:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/2be8a02f353f479984d8e2378c25ab95 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2be8a02f353f479984d8e2378c25ab95 2024-11-09T22:42:13,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2be8a02f353f479984d8e2378c25ab95, entries=150, sequenceid=122, filesize=11.7 K 2024-11-09T22:42:13,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0af038d6aa643579358dbb0f202b0f57 in 1681ms, sequenceid=122, compaction requested=true 2024-11-09T22:42:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:42:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:42:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:13,578 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:42:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:42:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:13,578 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:42:13,582 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:42:13,582 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/A is initiating minor compaction (all files) 2024-11-09T22:42:13,582 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:42:13,582 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/B is initiating minor compaction (all files) 2024-11-09T22:42:13,582 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/A in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:13,582 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/B in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:13,582 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/bcfec35b8ce04835aedcdca6c56d5f1c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96bb8790d4874382b6c5db27d9d1f749, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4d4f8d76e15d464780b84538b1fdf4bb, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4a4295d5b0b94708b5c51fa0e1b93c2a] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=47.0 K 2024-11-09T22:42:13,582 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/b14239b40d8647ca96e2889f818275c4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/347c6a2812704108ae424b792c3ad331, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/320011209020494395f26dd5061f4968, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/a4a59ee4e39c48f3abeb6f2f3818dfb4] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=146.2 K 2024-11-09T22:42:13,582 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:13,582 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/b14239b40d8647ca96e2889f818275c4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/347c6a2812704108ae424b792c3ad331, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/320011209020494395f26dd5061f4968, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/a4a59ee4e39c48f3abeb6f2f3818dfb4] 2024-11-09T22:42:13,583 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting bcfec35b8ce04835aedcdca6c56d5f1c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731192127845 2024-11-09T22:42:13,583 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b14239b40d8647ca96e2889f818275c4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731192127845 2024-11-09T22:42:13,583 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 96bb8790d4874382b6c5db27d9d1f749, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731192127906 2024-11-09T22:42:13,583 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4f8d76e15d464780b84538b1fdf4bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731192129051 2024-11-09T22:42:13,583 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 347c6a2812704108ae424b792c3ad331, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731192127904 2024-11-09T22:42:13,583 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a4295d5b0b94708b5c51fa0e1b93c2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731192131243 2024-11-09T22:42:13,583 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 320011209020494395f26dd5061f4968, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731192129051 2024-11-09T22:42:13,584 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a59ee4e39c48f3abeb6f2f3818dfb4, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731192131243 2024-11-09T22:42:13,588 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:13,590 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:13,590 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411091969016871654d4296e67e9e6e09f6ea_0af038d6aa643579358dbb0f202b0f57 store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:13,590 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/efe8975b713042c2b515540659c923d3 is 50, key is test_row_0/B:col10/1731192131897/Put/seqid=0 2024-11-09T22:42:13,592 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411091969016871654d4296e67e9e6e09f6ea_0af038d6aa643579358dbb0f202b0f57, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:13,592 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091969016871654d4296e67e9e6e09f6ea_0af038d6aa643579358dbb0f202b0f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:13,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742336_1512 (size=4469) 2024-11-09T22:42:13,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742335_1511 (size=12241) 2024-11-09T22:42:13,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-09T22:42:13,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:13,652 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T22:42:13,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091594221c68e14cfb919e94dc8d416189_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192131907/Put/seqid=0 2024-11-09T22:42:13,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742337_1513 (size=12254) 2024-11-09T22:42:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T22:42:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T22:42:14,005 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#A#compaction#435 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:14,005 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/32e3a83fe66b4ce2a8f367d5133b8321 is 175, key is test_row_0/A:col10/1731192131897/Put/seqid=0 2024-11-09T22:42:14,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742338_1514 (size=31195) 2024-11-09T22:42:14,014 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/efe8975b713042c2b515540659c923d3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/efe8975b713042c2b515540659c923d3 2024-11-09T22:42:14,018 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/B of 0af038d6aa643579358dbb0f202b0f57 into efe8975b713042c2b515540659c923d3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:14,018 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:14,018 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/B, priority=12, startTime=1731192133578; duration=0sec 2024-11-09T22:42:14,018 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:14,018 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:B 2024-11-09T22:42:14,018 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:42:14,020 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:42:14,020 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/C is initiating minor compaction (all files) 2024-11-09T22:42:14,020 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/C in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:14,020 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/24ffce2873ba49ee8e7db09620a55a43, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/29243fdd264b4ff49cc0189bf650a5ea, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/a9c8dbf74307484db2a0d19500f8dbee, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2be8a02f353f479984d8e2378c25ab95] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=47.0 K 2024-11-09T22:42:14,020 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ffce2873ba49ee8e7db09620a55a43, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731192127845 2024-11-09T22:42:14,020 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 29243fdd264b4ff49cc0189bf650a5ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731192127906 2024-11-09T22:42:14,021 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c8dbf74307484db2a0d19500f8dbee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731192129051 2024-11-09T22:42:14,021 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be8a02f353f479984d8e2378c25ab95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731192131243 2024-11-09T22:42:14,027 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:14,027 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/250e62ee65d941f0be8d32e1aa66da2f is 50, key is test_row_0/C:col10/1731192131897/Put/seqid=0 2024-11-09T22:42:14,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742339_1515 (size=12241) 2024-11-09T22:42:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:14,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:14,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:14,067 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091594221c68e14cfb919e94dc8d416189_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091594221c68e14cfb919e94dc8d416189_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:14,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/f83d98ecf0734c4aadbdcf010e8be7d3, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:14,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/f83d98ecf0734c4aadbdcf010e8be7d3 is 175, key is test_row_0/A:col10/1731192131907/Put/seqid=0 2024-11-09T22:42:14,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742340_1516 (size=31055) 2024-11-09T22:42:14,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192194104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192194105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192194106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192194215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192194215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192194215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,413 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/32e3a83fe66b4ce2a8f367d5133b8321 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/32e3a83fe66b4ce2a8f367d5133b8321 2024-11-09T22:42:14,416 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/A of 0af038d6aa643579358dbb0f202b0f57 into 32e3a83fe66b4ce2a8f367d5133b8321(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:14,416 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:14,416 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/A, priority=12, startTime=1731192133578; duration=0sec 2024-11-09T22:42:14,416 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:14,416 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:A 2024-11-09T22:42:14,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192194425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192194425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192194425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,439 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/250e62ee65d941f0be8d32e1aa66da2f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/250e62ee65d941f0be8d32e1aa66da2f 2024-11-09T22:42:14,443 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/C of 0af038d6aa643579358dbb0f202b0f57 into 250e62ee65d941f0be8d32e1aa66da2f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:14,443 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:14,443 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/C, priority=12, startTime=1731192133578; duration=0sec 2024-11-09T22:42:14,443 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:14,443 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:C 2024-11-09T22:42:14,472 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/f83d98ecf0734c4aadbdcf010e8be7d3 2024-11-09T22:42:14,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/2f9ef7266a0046ab8b59f77c3db70805 is 50, key is test_row_0/B:col10/1731192131907/Put/seqid=0 2024-11-09T22:42:14,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742341_1517 (size=12101) 2024-11-09T22:42:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T22:42:14,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192194735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192194735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192194735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:14,881 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/2f9ef7266a0046ab8b59f77c3db70805 2024-11-09T22:42:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/12cd4cf692504f008e456831d7731d83 is 50, key is test_row_0/C:col10/1731192131907/Put/seqid=0 2024-11-09T22:42:14,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742342_1518 (size=12101) 2024-11-09T22:42:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192195245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192195245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192195245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:15,290 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/12cd4cf692504f008e456831d7731d83 2024-11-09T22:42:15,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/f83d98ecf0734c4aadbdcf010e8be7d3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/f83d98ecf0734c4aadbdcf010e8be7d3 2024-11-09T22:42:15,296 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/f83d98ecf0734c4aadbdcf010e8be7d3, entries=150, sequenceid=131, filesize=30.3 K 2024-11-09T22:42:15,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/2f9ef7266a0046ab8b59f77c3db70805 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/2f9ef7266a0046ab8b59f77c3db70805 2024-11-09T22:42:15,300 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/2f9ef7266a0046ab8b59f77c3db70805, entries=150, sequenceid=131, filesize=11.8 K 2024-11-09T22:42:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/12cd4cf692504f008e456831d7731d83 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/12cd4cf692504f008e456831d7731d83 2024-11-09T22:42:15,303 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/12cd4cf692504f008e456831d7731d83, entries=150, sequenceid=131, filesize=11.8 K 2024-11-09T22:42:15,304 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 0af038d6aa643579358dbb0f202b0f57 in 1653ms, sequenceid=131, compaction requested=false 2024-11-09T22:42:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-09T22:42:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-09T22:42:15,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-09T22:42:15,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9580 sec 2024-11-09T22:42:15,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9620 sec 2024-11-09T22:42:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-09T22:42:15,495 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:15,495 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:42:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-09T22:42:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T22:42:15,497 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:42:15,497 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:42:15,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:42:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T22:42:15,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-09T22:42:15,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:15,650 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T22:42:15,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090feb647ee1364ae690a9055aca9fe7ad_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192134104/Put/seqid=0 2024-11-09T22:42:15,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742343_1519 (size=12304) 2024-11-09T22:42:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T22:42:16,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:16,062 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090feb647ee1364ae690a9055aca9fe7ad_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090feb647ee1364ae690a9055aca9fe7ad_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/ae5c9086edc54143ba7823ed0c7d37af, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/ae5c9086edc54143ba7823ed0c7d37af is 175, key is test_row_0/A:col10/1731192134104/Put/seqid=0 2024-11-09T22:42:16,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742344_1520 (size=31105) 2024-11-09T22:42:16,066 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/ae5c9086edc54143ba7823ed0c7d37af 2024-11-09T22:42:16,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/6e5e4fca36684f91b7940a9bf4e92f2d is 50, key is test_row_0/B:col10/1731192134104/Put/seqid=0 2024-11-09T22:42:16,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742345_1521 (size=12151) 2024-11-09T22:42:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T22:42:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:16,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:16,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192196260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192196262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192196262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192196364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192196375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192196375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,476 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/6e5e4fca36684f91b7940a9bf4e92f2d 2024-11-09T22:42:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/d2b6fd723bfc46489a94b34b73b67235 is 50, key is test_row_0/C:col10/1731192134104/Put/seqid=0 2024-11-09T22:42:16,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742346_1522 (size=12151) 2024-11-09T22:42:16,488 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/d2b6fd723bfc46489a94b34b73b67235 2024-11-09T22:42:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/ae5c9086edc54143ba7823ed0c7d37af as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/ae5c9086edc54143ba7823ed0c7d37af 2024-11-09T22:42:16,496 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/ae5c9086edc54143ba7823ed0c7d37af, entries=150, sequenceid=161, filesize=30.4 K 2024-11-09T22:42:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/6e5e4fca36684f91b7940a9bf4e92f2d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/6e5e4fca36684f91b7940a9bf4e92f2d 2024-11-09T22:42:16,499 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/6e5e4fca36684f91b7940a9bf4e92f2d, entries=150, sequenceid=161, filesize=11.9 K 2024-11-09T22:42:16,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/d2b6fd723bfc46489a94b34b73b67235 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/d2b6fd723bfc46489a94b34b73b67235 2024-11-09T22:42:16,503 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/d2b6fd723bfc46489a94b34b73b67235, entries=150, sequenceid=161, filesize=11.9 K 2024-11-09T22:42:16,504 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0af038d6aa643579358dbb0f202b0f57 in 854ms, sequenceid=161, compaction requested=true 2024-11-09T22:42:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-09T22:42:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-09T22:42:16,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-09T22:42:16,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0080 sec 2024-11-09T22:42:16,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0120 sec 2024-11-09T22:42:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:16,579 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:42:16,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:42:16,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c71978034ffd4ad4b7492f13fb77d7df_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192136578/Put/seqid=0 2024-11-09T22:42:16,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742347_1523 (size=12304) 2024-11-09T22:42:16,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:16,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c71978034ffd4ad4b7492f13fb77d7df_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c71978034ffd4ad4b7492f13fb77d7df_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:16,591 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/4651af3662274d5985b27018dc7189b6, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:16,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/4651af3662274d5985b27018dc7189b6 is 175, key is test_row_0/A:col10/1731192136578/Put/seqid=0 2024-11-09T22:42:16,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742348_1524 (size=31105) 2024-11-09T22:42:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192196622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192196623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192196624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-09T22:42:16,635 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:16,636 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:42:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-09T22:42:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T22:42:16,637 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:42:16,637 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:42:16,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:42:16,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192196735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192196735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192196735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T22:42:16,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T22:42:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:16,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:16,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T22:42:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:16,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:16,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192196945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192196945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192196945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T22:42:16,994 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/4651af3662274d5985b27018dc7189b6 2024-11-09T22:42:16,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/172a8792d13642cf8b79811677774db2 is 50, key is test_row_0/B:col10/1731192136578/Put/seqid=0 2024-11-09T22:42:17,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742349_1525 (size=12151) 2024-11-09T22:42:17,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T22:42:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T22:42:17,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:17,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192197254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192197254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192197255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T22:42:17,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T22:42:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/172a8792d13642cf8b79811677774db2 2024-11-09T22:42:17,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/88edd9f1bb7d44c2805238a17e24eef2 is 50, key is test_row_0/C:col10/1731192136578/Put/seqid=0 2024-11-09T22:42:17,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742350_1526 (size=12151) 2024-11-09T22:42:17,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T22:42:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T22:42:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T22:42:17,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192197764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:17,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192197764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:17,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192197765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T22:42:17,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/88edd9f1bb7d44c2805238a17e24eef2 2024-11-09T22:42:17,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/4651af3662274d5985b27018dc7189b6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/4651af3662274d5985b27018dc7189b6 2024-11-09T22:42:17,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/4651af3662274d5985b27018dc7189b6, entries=150, sequenceid=172, filesize=30.4 K 2024-11-09T22:42:17,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/172a8792d13642cf8b79811677774db2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/172a8792d13642cf8b79811677774db2 2024-11-09T22:42:17,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/172a8792d13642cf8b79811677774db2, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T22:42:17,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/88edd9f1bb7d44c2805238a17e24eef2 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/88edd9f1bb7d44c2805238a17e24eef2 2024-11-09T22:42:17,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/88edd9f1bb7d44c2805238a17e24eef2, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T22:42:17,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0af038d6aa643579358dbb0f202b0f57 in 1256ms, sequenceid=172, compaction requested=true 2024-11-09T22:42:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:42:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:17,835 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:42:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:42:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:42:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:17,835 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:42:17,836 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:42:17,836 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:42:17,836 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/B is initiating minor compaction (all files) 2024-11-09T22:42:17,836 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/A is initiating minor compaction (all files) 2024-11-09T22:42:17,836 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/A in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,836 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/B in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,837 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/32e3a83fe66b4ce2a8f367d5133b8321, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/f83d98ecf0734c4aadbdcf010e8be7d3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/ae5c9086edc54143ba7823ed0c7d37af, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/4651af3662274d5985b27018dc7189b6] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=121.5 K 2024-11-09T22:42:17,837 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/efe8975b713042c2b515540659c923d3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/2f9ef7266a0046ab8b59f77c3db70805, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/6e5e4fca36684f91b7940a9bf4e92f2d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/172a8792d13642cf8b79811677774db2] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=47.5 K 2024-11-09T22:42:17,837 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,837 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/32e3a83fe66b4ce2a8f367d5133b8321, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/f83d98ecf0734c4aadbdcf010e8be7d3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/ae5c9086edc54143ba7823ed0c7d37af, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/4651af3662274d5985b27018dc7189b6] 2024-11-09T22:42:17,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting efe8975b713042c2b515540659c923d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731192131243 2024-11-09T22:42:17,837 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32e3a83fe66b4ce2a8f367d5133b8321, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731192131243 2024-11-09T22:42:17,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9ef7266a0046ab8b59f77c3db70805, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731192131901 2024-11-09T22:42:17,837 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83d98ecf0734c4aadbdcf010e8be7d3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731192131901 2024-11-09T22:42:17,837 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e5e4fca36684f91b7940a9bf4e92f2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731192134099 2024-11-09T22:42:17,838 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5c9086edc54143ba7823ed0c7d37af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731192134099 2024-11-09T22:42:17,838 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 172a8792d13642cf8b79811677774db2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192136261 2024-11-09T22:42:17,838 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4651af3662274d5985b27018dc7189b6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192136261 2024-11-09T22:42:17,843 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:17,844 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:17,844 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/21ddd042428a4113b165b52db953c533 is 50, key is test_row_0/B:col10/1731192136578/Put/seqid=0 2024-11-09T22:42:17,846 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109b2a3ac4dc4fc43f3919b68008c9acbea_0af038d6aa643579358dbb0f202b0f57 store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:17,848 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109b2a3ac4dc4fc43f3919b68008c9acbea_0af038d6aa643579358dbb0f202b0f57, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:17,848 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b2a3ac4dc4fc43f3919b68008c9acbea_0af038d6aa643579358dbb0f202b0f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:17,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742351_1527 (size=12527) 2024-11-09T22:42:17,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742352_1528 (size=4469) 2024-11-09T22:42:17,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-09T22:42:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:17,868 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:42:17,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110990121b188db14e7b9c642b00cfee6ae6_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192136616/Put/seqid=0 2024-11-09T22:42:17,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742353_1529 (size=12304) 2024-11-09T22:42:18,255 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/21ddd042428a4113b165b52db953c533 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/21ddd042428a4113b165b52db953c533 2024-11-09T22:42:18,258 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/B of 0af038d6aa643579358dbb0f202b0f57 into 21ddd042428a4113b165b52db953c533(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:18,258 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:18,258 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/B, priority=12, startTime=1731192137835; duration=0sec 2024-11-09T22:42:18,258 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:18,258 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:B 2024-11-09T22:42:18,258 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T22:42:18,259 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T22:42:18,259 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/C is initiating minor compaction (all files) 2024-11-09T22:42:18,259 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/C in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:18,259 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/250e62ee65d941f0be8d32e1aa66da2f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/12cd4cf692504f008e456831d7731d83, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/d2b6fd723bfc46489a94b34b73b67235, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/88edd9f1bb7d44c2805238a17e24eef2] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=47.5 K 2024-11-09T22:42:18,260 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 250e62ee65d941f0be8d32e1aa66da2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731192131243 2024-11-09T22:42:18,260 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 12cd4cf692504f008e456831d7731d83, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731192131901 2024-11-09T22:42:18,260 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b6fd723bfc46489a94b34b73b67235, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731192134099 2024-11-09T22:42:18,260 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 88edd9f1bb7d44c2805238a17e24eef2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192136261 2024-11-09T22:42:18,261 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#A#compaction#447 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:18,262 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/2ebfe6e8c86141df9bb8deb7abf93787 is 175, key is test_row_0/A:col10/1731192136578/Put/seqid=0 2024-11-09T22:42:18,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742354_1530 (size=31481) 2024-11-09T22:42:18,267 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:18,267 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/3e4b1259e6914c408798f208bdf9d6f1 is 50, key is test_row_0/C:col10/1731192136578/Put/seqid=0 2024-11-09T22:42:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742355_1531 (size=12527) 2024-11-09T22:42:18,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:18,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110990121b188db14e7b9c642b00cfee6ae6_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110990121b188db14e7b9c642b00cfee6ae6_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:18,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/0eff74cd9ea44824b528e83bd9ac25db, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/0eff74cd9ea44824b528e83bd9ac25db is 175, key is test_row_0/A:col10/1731192136616/Put/seqid=0 2024-11-09T22:42:18,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742356_1532 (size=31105) 2024-11-09T22:42:18,669 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/2ebfe6e8c86141df9bb8deb7abf93787 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/2ebfe6e8c86141df9bb8deb7abf93787 2024-11-09T22:42:18,673 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/A of 0af038d6aa643579358dbb0f202b0f57 into 2ebfe6e8c86141df9bb8deb7abf93787(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:18,673 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:18,673 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/A, priority=12, startTime=1731192137835; duration=0sec 2024-11-09T22:42:18,673 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:18,673 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:A 2024-11-09T22:42:18,674 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/3e4b1259e6914c408798f208bdf9d6f1 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/3e4b1259e6914c408798f208bdf9d6f1 2024-11-09T22:42:18,677 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/C of 0af038d6aa643579358dbb0f202b0f57 into 3e4b1259e6914c408798f208bdf9d6f1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:18,677 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:18,677 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/C, priority=12, startTime=1731192137835; duration=0sec 2024-11-09T22:42:18,677 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:18,678 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:C 2024-11-09T22:42:18,686 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/0eff74cd9ea44824b528e83bd9ac25db 2024-11-09T22:42:18,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/57e20006f81c4078bbbff9c639dfcb3e is 50, key is test_row_0/B:col10/1731192136616/Put/seqid=0 2024-11-09T22:42:18,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742357_1533 (size=12151) 2024-11-09T22:42:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:18,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T22:42:18,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192198788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:18,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192198790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:18,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192198791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:18,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192198895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192198895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:18,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192198904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:18,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:18,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,095 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/57e20006f81c4078bbbff9c639dfcb3e 2024-11-09T22:42:19,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/f048f3a40763468187deb0f7ed5d8c53 is 50, key is test_row_0/C:col10/1731192136616/Put/seqid=0 2024-11-09T22:42:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742358_1534 (size=12151) 2024-11-09T22:42:19,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192199105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192199105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192199115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192199414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192199414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192199425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/f048f3a40763468187deb0f7ed5d8c53 2024-11-09T22:42:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/0eff74cd9ea44824b528e83bd9ac25db as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/0eff74cd9ea44824b528e83bd9ac25db 2024-11-09T22:42:19,511 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/0eff74cd9ea44824b528e83bd9ac25db, entries=150, sequenceid=197, filesize=30.4 K 2024-11-09T22:42:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/57e20006f81c4078bbbff9c639dfcb3e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/57e20006f81c4078bbbff9c639dfcb3e 2024-11-09T22:42:19,514 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/57e20006f81c4078bbbff9c639dfcb3e, entries=150, sequenceid=197, filesize=11.9 K 2024-11-09T22:42:19,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/f048f3a40763468187deb0f7ed5d8c53 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/f048f3a40763468187deb0f7ed5d8c53 2024-11-09T22:42:19,518 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/f048f3a40763468187deb0f7ed5d8c53, entries=150, sequenceid=197, filesize=11.9 K 2024-11-09T22:42:19,519 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0af038d6aa643579358dbb0f202b0f57 in 1651ms, sequenceid=197, compaction requested=false 2024-11-09T22:42:19,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:19,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:19,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-09T22:42:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-09T22:42:19,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-09T22:42:19,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8830 sec 2024-11-09T22:42:19,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.8860 sec 2024-11-09T22:42:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:19,927 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:42:19,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:42:19,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c5d5b8ef213143559c7299fe04145ade_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192138786/Put/seqid=0 2024-11-09T22:42:19,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742359_1535 (size=14794) 2024-11-09T22:42:19,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192199988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192199988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192199988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192200095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192200095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192200095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192200305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192200305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192200305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,338 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:20,341 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c5d5b8ef213143559c7299fe04145ade_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c5d5b8ef213143559c7299fe04145ade_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:20,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/755ce70a10db48d3a6f646ad7f1b5559, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:20,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/755ce70a10db48d3a6f646ad7f1b5559 is 175, key is test_row_0/A:col10/1731192138786/Put/seqid=0 2024-11-09T22:42:20,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742360_1536 (size=39749) 2024-11-09T22:42:20,346 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/755ce70a10db48d3a6f646ad7f1b5559 2024-11-09T22:42:20,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/c6cb3e6b2e2747a4b3ed097b880d820b is 50, key is test_row_0/B:col10/1731192138786/Put/seqid=0 2024-11-09T22:42:20,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742361_1537 (size=12151) 2024-11-09T22:42:20,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192200615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192200615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192200615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:20,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/c6cb3e6b2e2747a4b3ed097b880d820b 2024-11-09T22:42:20,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/38ee0b15b9f840cdad9c2ef96292a65f is 50, key is test_row_0/C:col10/1731192138786/Put/seqid=0 2024-11-09T22:42:20,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742362_1538 (size=12151) 2024-11-09T22:42:20,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/38ee0b15b9f840cdad9c2ef96292a65f 2024-11-09T22:42:20,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/755ce70a10db48d3a6f646ad7f1b5559 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/755ce70a10db48d3a6f646ad7f1b5559 2024-11-09T22:42:20,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/755ce70a10db48d3a6f646ad7f1b5559, entries=200, sequenceid=213, filesize=38.8 K 2024-11-09T22:42:20,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/c6cb3e6b2e2747a4b3ed097b880d820b as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c6cb3e6b2e2747a4b3ed097b880d820b 2024-11-09T22:42:20,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c6cb3e6b2e2747a4b3ed097b880d820b, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T22:42:20,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/38ee0b15b9f840cdad9c2ef96292a65f as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/38ee0b15b9f840cdad9c2ef96292a65f 2024-11-09T22:42:20,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/38ee0b15b9f840cdad9c2ef96292a65f, entries=150, sequenceid=213, filesize=11.9 K 2024-11-09T22:42:20,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 0af038d6aa643579358dbb0f202b0f57 in 852ms, sequenceid=213, compaction requested=true 2024-11-09T22:42:20,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:20,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:42:20,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:20,779 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:42:20,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:42:20,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:20,779 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:42:20,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:42:20,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:20,780 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:42:20,780 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:42:20,780 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/B is initiating minor compaction (all files) 2024-11-09T22:42:20,780 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/A is initiating minor compaction (all files) 2024-11-09T22:42:20,780 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/B in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:20,780 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/A in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:20,780 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/21ddd042428a4113b165b52db953c533, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/57e20006f81c4078bbbff9c639dfcb3e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c6cb3e6b2e2747a4b3ed097b880d820b] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=36.0 K 2024-11-09T22:42:20,780 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/2ebfe6e8c86141df9bb8deb7abf93787, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/0eff74cd9ea44824b528e83bd9ac25db, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/755ce70a10db48d3a6f646ad7f1b5559] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=99.9 K 2024-11-09T22:42:20,780 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:20,781 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/2ebfe6e8c86141df9bb8deb7abf93787, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/0eff74cd9ea44824b528e83bd9ac25db, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/755ce70a10db48d3a6f646ad7f1b5559] 2024-11-09T22:42:20,781 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 21ddd042428a4113b165b52db953c533, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192136261 2024-11-09T22:42:20,781 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ebfe6e8c86141df9bb8deb7abf93787, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192136261 2024-11-09T22:42:20,782 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e20006f81c4078bbbff9c639dfcb3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731192136616 2024-11-09T22:42:20,782 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eff74cd9ea44824b528e83bd9ac25db, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731192136616 2024-11-09T22:42:20,782 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting c6cb3e6b2e2747a4b3ed097b880d820b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731192138786 2024-11-09T22:42:20,782 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 755ce70a10db48d3a6f646ad7f1b5559, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731192138786 2024-11-09T22:42:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T22:42:20,795 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:20,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:42:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-09T22:42:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T22:42:20,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:42:20,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:42:20,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:42:20,803 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:20,805 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:20,805 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109c4acda0df4194760882556ccada6c299_0af038d6aa643579358dbb0f202b0f57 store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:20,805 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/65cdc5cd892f47d9a44d6825443f15be is 50, key is test_row_0/B:col10/1731192138786/Put/seqid=0 2024-11-09T22:42:20,807 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109c4acda0df4194760882556ccada6c299_0af038d6aa643579358dbb0f202b0f57, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:20,808 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c4acda0df4194760882556ccada6c299_0af038d6aa643579358dbb0f202b0f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:20,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742363_1539 (size=12629) 2024-11-09T22:42:20,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742364_1540 (size=4469) 2024-11-09T22:42:20,812 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#A#compaction#456 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:20,812 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/95ee4884c77447f8a0755cd0face6cff is 175, key is test_row_0/A:col10/1731192138786/Put/seqid=0 2024-11-09T22:42:20,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742365_1541 (size=31583) 2024-11-09T22:42:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T22:42:20,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-09T22:42:20,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:20,951 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:42:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ffeb228fad7b4ee683977ce0c32b21ac_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192139966/Put/seqid=0 2024-11-09T22:42:20,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742366_1542 (size=12304) 2024-11-09T22:42:20,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:20,964 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ffeb228fad7b4ee683977ce0c32b21ac_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ffeb228fad7b4ee683977ce0c32b21ac_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:20,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/55e2d35544da4532b995280c5024622a, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/55e2d35544da4532b995280c5024622a is 175, key is test_row_0/A:col10/1731192139966/Put/seqid=0 2024-11-09T22:42:20,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742367_1543 (size=31105) 2024-11-09T22:42:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T22:42:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:21,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192201133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192201136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192201138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,212 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/65cdc5cd892f47d9a44d6825443f15be as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/65cdc5cd892f47d9a44d6825443f15be 2024-11-09T22:42:21,216 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/B of 0af038d6aa643579358dbb0f202b0f57 into 65cdc5cd892f47d9a44d6825443f15be(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:21,216 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:21,216 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/B, priority=13, startTime=1731192140779; duration=0sec 2024-11-09T22:42:21,216 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:21,216 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:B 2024-11-09T22:42:21,216 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T22:42:21,217 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T22:42:21,217 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/C is initiating minor compaction (all files) 2024-11-09T22:42:21,217 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/C in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:21,217 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/3e4b1259e6914c408798f208bdf9d6f1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/f048f3a40763468187deb0f7ed5d8c53, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/38ee0b15b9f840cdad9c2ef96292a65f] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=36.0 K 2024-11-09T22:42:21,217 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4b1259e6914c408798f208bdf9d6f1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731192136261 2024-11-09T22:42:21,217 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting f048f3a40763468187deb0f7ed5d8c53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731192136616 2024-11-09T22:42:21,218 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ee0b15b9f840cdad9c2ef96292a65f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731192138786 2024-11-09T22:42:21,222 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/95ee4884c77447f8a0755cd0face6cff as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/95ee4884c77447f8a0755cd0face6cff 2024-11-09T22:42:21,224 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#C#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:21,224 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/193f0999062348979f8b572934a53fc3 is 50, key is test_row_0/C:col10/1731192138786/Put/seqid=0 2024-11-09T22:42:21,226 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/A of 0af038d6aa643579358dbb0f202b0f57 into 95ee4884c77447f8a0755cd0face6cff(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:21,226 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:21,227 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/A, priority=13, startTime=1731192140779; duration=0sec 2024-11-09T22:42:21,227 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:21,227 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:A 2024-11-09T22:42:21,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742368_1544 (size=12629) 2024-11-09T22:42:21,238 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/193f0999062348979f8b572934a53fc3 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/193f0999062348979f8b572934a53fc3 2024-11-09T22:42:21,241 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/C of 0af038d6aa643579358dbb0f202b0f57 into 193f0999062348979f8b572934a53fc3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:21,241 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:21,241 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/C, priority=13, startTime=1731192140779; duration=0sec 2024-11-09T22:42:21,241 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:21,241 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:C 2024-11-09T22:42:21,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192201245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192201245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192201245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,369 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/55e2d35544da4532b995280c5024622a 2024-11-09T22:42:21,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/f6933c40e21e414bac456463b6ff5048 is 50, key is test_row_0/B:col10/1731192139966/Put/seqid=0 2024-11-09T22:42:21,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742369_1545 (size=12151) 2024-11-09T22:42:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T22:42:21,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192201455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192201455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192201455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192201765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192201765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192201765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:21,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:21,778 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/f6933c40e21e414bac456463b6ff5048 2024-11-09T22:42:21,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/359e246d912249f998957e25977bf46e is 50, key is test_row_0/C:col10/1731192139966/Put/seqid=0 2024-11-09T22:42:21,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742370_1546 (size=12151) 2024-11-09T22:42:21,793 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/359e246d912249f998957e25977bf46e 2024-11-09T22:42:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/55e2d35544da4532b995280c5024622a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/55e2d35544da4532b995280c5024622a 2024-11-09T22:42:21,799 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/55e2d35544da4532b995280c5024622a, entries=150, sequenceid=238, filesize=30.4 K 2024-11-09T22:42:21,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/f6933c40e21e414bac456463b6ff5048 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/f6933c40e21e414bac456463b6ff5048 2024-11-09T22:42:21,804 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/f6933c40e21e414bac456463b6ff5048, entries=150, sequenceid=238, filesize=11.9 K 2024-11-09T22:42:21,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/359e246d912249f998957e25977bf46e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/359e246d912249f998957e25977bf46e 2024-11-09T22:42:21,808 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/359e246d912249f998957e25977bf46e, entries=150, sequenceid=238, filesize=11.9 K 2024-11-09T22:42:21,808 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0af038d6aa643579358dbb0f202b0f57 in 857ms, sequenceid=238, compaction requested=false 2024-11-09T22:42:21,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:21,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:21,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-09T22:42:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-09T22:42:21,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-09T22:42:21,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0110 sec 2024-11-09T22:42:21,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0160 sec 2024-11-09T22:42:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-09T22:42:21,935 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:21,935 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:42:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-09T22:42:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T22:42:21,937 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:42:21,938 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:42:21,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:42:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T22:42:22,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-09T22:42:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:22,090 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T22:42:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110963ec61807a29429188a5eac59620d879_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192141137/Put/seqid=0 2024-11-09T22:42:22,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742371_1547 (size=12304) 2024-11-09T22:42:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T22:42:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:22,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192202305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192202307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192202308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192202415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192202415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192202415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:22,548 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110963ec61807a29429188a5eac59620d879_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963ec61807a29429188a5eac59620d879_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:22,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/46cc709a88fc4f4d9d596672df3f5503, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:22,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/46cc709a88fc4f4d9d596672df3f5503 is 175, key is test_row_0/A:col10/1731192141137/Put/seqid=0 2024-11-09T22:42:22,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742372_1548 (size=31105) 2024-11-09T22:42:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T22:42:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192202625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192202625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192202625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192202934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192202935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192202935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:22,952 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/46cc709a88fc4f4d9d596672df3f5503 2024-11-09T22:42:22,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/53c94092944242d2941dfbf8ee0ca1b0 is 50, key is test_row_0/B:col10/1731192141137/Put/seqid=0 2024-11-09T22:42:22,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742373_1549 (size=12151) 2024-11-09T22:42:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T22:42:23,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60746 deadline: 1731192203265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:23,269 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_1' in 0af038d6aa643579358dbb0f202b0f57 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:42:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60726 deadline: 1731192203345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:23,347 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d8c37854f6e,35101,1731191981027 for 'test_row_0' in 0af038d6aa643579358dbb0f202b0f57 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-09T22:42:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:23,360 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/53c94092944242d2941dfbf8ee0ca1b0 2024-11-09T22:42:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/6453ac73477b46b192138c59c6e4f5be is 50, key is test_row_0/C:col10/1731192141137/Put/seqid=0 2024-11-09T22:42:23,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742374_1550 (size=12151) 2024-11-09T22:42:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192203445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192203445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192203445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:23,770 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/6453ac73477b46b192138c59c6e4f5be 2024-11-09T22:42:23,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/46cc709a88fc4f4d9d596672df3f5503 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/46cc709a88fc4f4d9d596672df3f5503 2024-11-09T22:42:23,782 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/46cc709a88fc4f4d9d596672df3f5503, entries=150, sequenceid=252, filesize=30.4 K 2024-11-09T22:42:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/53c94092944242d2941dfbf8ee0ca1b0 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/53c94092944242d2941dfbf8ee0ca1b0 2024-11-09T22:42:23,785 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/53c94092944242d2941dfbf8ee0ca1b0, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T22:42:23,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/6453ac73477b46b192138c59c6e4f5be as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/6453ac73477b46b192138c59c6e4f5be 2024-11-09T22:42:23,791 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/6453ac73477b46b192138c59c6e4f5be, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T22:42:23,792 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0af038d6aa643579358dbb0f202b0f57 in 1702ms, sequenceid=252, compaction requested=true 2024-11-09T22:42:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-09T22:42:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-09T22:42:23,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-09T22:42:23,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8570 sec 2024-11-09T22:42:23,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8620 sec 2024-11-09T22:42:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-09T22:42:24,085 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:24,085 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T22:42:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-09T22:42:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T22:42:24,087 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T22:42:24,087 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T22:42:24,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T22:42:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T22:42:24,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-09T22:42:24,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:24,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T22:42:24,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dfc4ad44fc084fc7a6a64e355dcca575_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192142306/Put/seqid=0 2024-11-09T22:42:24,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742375_1551 (size=12454) 2024-11-09T22:42:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T22:42:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:24,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. as already flushing 2024-11-09T22:42:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192204466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:24,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192204467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:24,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192204470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192204574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192204574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:24,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192204585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:24,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:24,660 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dfc4ad44fc084fc7a6a64e355dcca575_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dfc4ad44fc084fc7a6a64e355dcca575_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/49df6dae045446d6b24f0cdd4b454a7e, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/49df6dae045446d6b24f0cdd4b454a7e is 175, key is test_row_0/A:col10/1731192142306/Put/seqid=0 2024-11-09T22:42:24,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742376_1552 (size=31255) 2024-11-09T22:42:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T22:42:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192204785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192204785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:24,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192204794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:25,036 INFO [Thread-2186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2186. 2024-11-09T22:42:25,036 DEBUG [Thread-2186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:25,036 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,036 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:25,037 INFO [Thread-2184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2184. 2024-11-09T22:42:25,037 DEBUG [Thread-2184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:25,037 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,037 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:25,038 INFO [Thread-2182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2182. 2024-11-09T22:42:25,038 DEBUG [Thread-2182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:25,038 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,038 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:25,039 INFO [Thread-2180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2180. 2024-11-09T22:42:25,039 DEBUG [Thread-2180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:25,039 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,039 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:25,040 INFO [Thread-2178 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2178. 2024-11-09T22:42:25,040 DEBUG [Thread-2178 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:25,040 DEBUG [Thread-2178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,040 DEBUG [Thread-2178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:25,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:25,064 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/49df6dae045446d6b24f0cdd4b454a7e 2024-11-09T22:42:25,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/92fcd8f44b854cb7acb41ea231a66ad6 is 50, key is test_row_0/B:col10/1731192142306/Put/seqid=0 2024-11-09T22:42:25,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742377_1553 (size=12301) 2024-11-09T22:42:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192205095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192205095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:25,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192205105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T22:42:25,479 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/92fcd8f44b854cb7acb41ea231a66ad6 2024-11-09T22:42:25,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/ed602da0017046e2997f7478679d5ff0 is 50, key is test_row_0/C:col10/1731192142306/Put/seqid=0 2024-11-09T22:42:25,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742378_1554 (size=12301) 2024-11-09T22:42:25,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:25,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60732 deadline: 1731192205606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60758 deadline: 1731192205606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:25,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T22:42:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60702 deadline: 1731192205615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 , the old value is region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T22:42:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., hostname=9d8c37854f6e,35101,1731191981027, seqNum=5 because the exception is null or not the one we care about 2024-11-09T22:42:25,898 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/ed602da0017046e2997f7478679d5ff0 2024-11-09T22:42:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/49df6dae045446d6b24f0cdd4b454a7e as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/49df6dae045446d6b24f0cdd4b454a7e 2024-11-09T22:42:25,916 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/49df6dae045446d6b24f0cdd4b454a7e, entries=150, sequenceid=277, filesize=30.5 K 2024-11-09T22:42:25,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/92fcd8f44b854cb7acb41ea231a66ad6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/92fcd8f44b854cb7acb41ea231a66ad6 2024-11-09T22:42:25,921 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/92fcd8f44b854cb7acb41ea231a66ad6, entries=150, sequenceid=277, filesize=12.0 K 2024-11-09T22:42:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/ed602da0017046e2997f7478679d5ff0 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/ed602da0017046e2997f7478679d5ff0 2024-11-09T22:42:25,925 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/ed602da0017046e2997f7478679d5ff0, entries=150, sequenceid=277, filesize=12.0 K 2024-11-09T22:42:25,926 INFO [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0af038d6aa643579358dbb0f202b0f57 in 1686ms, sequenceid=277, compaction requested=true 2024-11-09T22:42:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d8c37854f6e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-09T22:42:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-09T22:42:25,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-09T22:42:25,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8420 sec 2024-11-09T22:42:25,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8470 sec 2024-11-09T22:42:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-09T22:42:26,246 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35101 {}] regionserver.HRegion(8855): Flush requested on 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:26,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-09T22:42:26,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T22:42:26,618 INFO [Thread-2169 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2169. 2024-11-09T22:42:26,618 DEBUG [Thread-2169 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:26,618 DEBUG [Thread-2169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:26,618 DEBUG [Thread-2169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:26,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:26,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110900c05e5fa22f47c5949e9d123676b96b_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192144465/Put/seqid=0 2024-11-09T22:42:26,627 INFO [Thread-2175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2175. 2024-11-09T22:42:26,627 DEBUG [Thread-2175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:26,627 DEBUG [Thread-2175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:26,627 INFO [Thread-2167 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2167. 2024-11-09T22:42:26,627 DEBUG [Thread-2167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:26,627 DEBUG [Thread-2167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:26,627 DEBUG [Thread-2175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:26,627 DEBUG [Thread-2167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:26,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:26,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:26,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742379_1555 (size=12454) 2024-11-09T22:42:27,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:27,042 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110900c05e5fa22f47c5949e9d123676b96b_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110900c05e5fa22f47c5949e9d123676b96b_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:27,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/99ea495252234ab68b1ddba3b3046665, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:27,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/99ea495252234ab68b1ddba3b3046665 is 175, key is test_row_0/A:col10/1731192144465/Put/seqid=0 2024-11-09T22:42:27,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742380_1556 (size=31255) 2024-11-09T22:42:27,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/99ea495252234ab68b1ddba3b3046665 2024-11-09T22:42:27,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/be4b57778c28499b9ace9eb1f96758f6 is 50, key is test_row_0/B:col10/1731192144465/Put/seqid=0 2024-11-09T22:42:27,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742381_1557 (size=12301) 2024-11-09T22:42:27,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/be4b57778c28499b9ace9eb1f96758f6 2024-11-09T22:42:27,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/8c58d2a2f45747a9a7c01685f5b993b9 is 50, key is test_row_0/C:col10/1731192144465/Put/seqid=0 2024-11-09T22:42:27,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742382_1558 (size=12301) 2024-11-09T22:42:28,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/8c58d2a2f45747a9a7c01685f5b993b9 2024-11-09T22:42:28,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/99ea495252234ab68b1ddba3b3046665 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/99ea495252234ab68b1ddba3b3046665 2024-11-09T22:42:28,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/99ea495252234ab68b1ddba3b3046665, entries=150, sequenceid=289, filesize=30.5 K 2024-11-09T22:42:28,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/be4b57778c28499b9ace9eb1f96758f6 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/be4b57778c28499b9ace9eb1f96758f6 2024-11-09T22:42:28,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/be4b57778c28499b9ace9eb1f96758f6, entries=150, sequenceid=289, filesize=12.0 K 2024-11-09T22:42:28,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/8c58d2a2f45747a9a7c01685f5b993b9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/8c58d2a2f45747a9a7c01685f5b993b9 2024-11-09T22:42:28,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/8c58d2a2f45747a9a7c01685f5b993b9, entries=150, sequenceid=289, filesize=12.0 K 2024-11-09T22:42:28,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=13.42 KB/13740 for 0af038d6aa643579358dbb0f202b0f57 in 1742ms, sequenceid=289, compaction requested=true 2024-11-09T22:42:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T22:42:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T22:42:28,359 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T22:42:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0af038d6aa643579358dbb0f202b0f57:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T22:42:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:28,359 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T22:42:28,361 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61533 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T22:42:28,361 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156303 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T22:42:28,361 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/A is initiating minor compaction (all files) 2024-11-09T22:42:28,361 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/B is initiating minor compaction (all files) 2024-11-09T22:42:28,361 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/A in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:28,361 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/B in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:28,361 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/65cdc5cd892f47d9a44d6825443f15be, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/f6933c40e21e414bac456463b6ff5048, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/53c94092944242d2941dfbf8ee0ca1b0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/92fcd8f44b854cb7acb41ea231a66ad6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/be4b57778c28499b9ace9eb1f96758f6] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=60.1 K 2024-11-09T22:42:28,361 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/95ee4884c77447f8a0755cd0face6cff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/55e2d35544da4532b995280c5024622a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/46cc709a88fc4f4d9d596672df3f5503, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/49df6dae045446d6b24f0cdd4b454a7e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/99ea495252234ab68b1ddba3b3046665] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=152.6 K 2024-11-09T22:42:28,361 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:28,361 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. files: [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/95ee4884c77447f8a0755cd0face6cff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/55e2d35544da4532b995280c5024622a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/46cc709a88fc4f4d9d596672df3f5503, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/49df6dae045446d6b24f0cdd4b454a7e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/99ea495252234ab68b1ddba3b3046665] 2024-11-09T22:42:28,362 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 65cdc5cd892f47d9a44d6825443f15be, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731192138786 2024-11-09T22:42:28,362 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ee4884c77447f8a0755cd0face6cff, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731192138786 2024-11-09T22:42:28,362 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting f6933c40e21e414bac456463b6ff5048, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731192139952 2024-11-09T22:42:28,362 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e2d35544da4532b995280c5024622a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731192139952 2024-11-09T22:42:28,362 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c94092944242d2941dfbf8ee0ca1b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731192141132 2024-11-09T22:42:28,362 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46cc709a88fc4f4d9d596672df3f5503, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731192141132 2024-11-09T22:42:28,362 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 92fcd8f44b854cb7acb41ea231a66ad6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731192142301 2024-11-09T22:42:28,362 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49df6dae045446d6b24f0cdd4b454a7e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731192142301 2024-11-09T22:42:28,363 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting be4b57778c28499b9ace9eb1f96758f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731192144465 2024-11-09T22:42:28,363 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99ea495252234ab68b1ddba3b3046665, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731192144465 2024-11-09T22:42:28,372 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:28,373 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411093000771481894ed2bba62b1759e03df6_0af038d6aa643579358dbb0f202b0f57 store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:28,374 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#B#compaction#470 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:28,374 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/b7e6a25d96b048cc8b9d6e3d5422c575 is 50, key is test_row_0/B:col10/1731192144465/Put/seqid=0 2024-11-09T22:42:28,376 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411093000771481894ed2bba62b1759e03df6_0af038d6aa643579358dbb0f202b0f57, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:28,377 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093000771481894ed2bba62b1759e03df6_0af038d6aa643579358dbb0f202b0f57 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:28,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742383_1559 (size=12949) 2024-11-09T22:42:28,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742384_1560 (size=4469) 2024-11-09T22:42:28,782 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#A#compaction#471 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:28,783 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/d340a2343d5e411385df612073e65c32 is 175, key is test_row_0/A:col10/1731192144465/Put/seqid=0 2024-11-09T22:42:28,788 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/b7e6a25d96b048cc8b9d6e3d5422c575 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/b7e6a25d96b048cc8b9d6e3d5422c575 2024-11-09T22:42:28,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742385_1561 (size=31903) 2024-11-09T22:42:28,794 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/B of 0af038d6aa643579358dbb0f202b0f57 into b7e6a25d96b048cc8b9d6e3d5422c575(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:28,794 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:28,794 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/B, priority=11, startTime=1731192148359; duration=0sec 2024-11-09T22:42:28,794 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T22:42:28,794 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:B 2024-11-09T22:42:28,794 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T22:42:28,795 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61533 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T22:42:28,795 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1541): 0af038d6aa643579358dbb0f202b0f57/C is initiating minor compaction (all files) 2024-11-09T22:42:28,795 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0af038d6aa643579358dbb0f202b0f57/C in TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:28,795 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/193f0999062348979f8b572934a53fc3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/359e246d912249f998957e25977bf46e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/6453ac73477b46b192138c59c6e4f5be, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/ed602da0017046e2997f7478679d5ff0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/8c58d2a2f45747a9a7c01685f5b993b9] into tmpdir=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp, totalSize=60.1 K 2024-11-09T22:42:28,796 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 193f0999062348979f8b572934a53fc3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731192138786 2024-11-09T22:42:28,796 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 359e246d912249f998957e25977bf46e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731192139952 2024-11-09T22:42:28,796 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6453ac73477b46b192138c59c6e4f5be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731192141132 2024-11-09T22:42:28,796 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting ed602da0017046e2997f7478679d5ff0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731192142301 2024-11-09T22:42:28,796 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c58d2a2f45747a9a7c01685f5b993b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731192144465 2024-11-09T22:42:28,803 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0af038d6aa643579358dbb0f202b0f57#C#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-09T22:42:28,804 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/2d927d5182f842bbaf23912d2dd5ac2d is 50, key is test_row_0/C:col10/1731192144465/Put/seqid=0 2024-11-09T22:42:28,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742386_1562 (size=12949) 2024-11-09T22:42:29,201 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/d340a2343d5e411385df612073e65c32 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/d340a2343d5e411385df612073e65c32 2024-11-09T22:42:29,206 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/A of 0af038d6aa643579358dbb0f202b0f57 into d340a2343d5e411385df612073e65c32(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:29,206 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:29,206 INFO [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/A, priority=11, startTime=1731192148359; duration=0sec 2024-11-09T22:42:29,206 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:29,206 DEBUG [RS:0;9d8c37854f6e:35101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:A 2024-11-09T22:42:29,210 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/2d927d5182f842bbaf23912d2dd5ac2d as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2d927d5182f842bbaf23912d2dd5ac2d 2024-11-09T22:42:29,216 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0af038d6aa643579358dbb0f202b0f57/C of 0af038d6aa643579358dbb0f202b0f57 into 2d927d5182f842bbaf23912d2dd5ac2d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T22:42:29,216 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0af038d6aa643579358dbb0f202b0f57: 2024-11-09T22:42:29,216 INFO [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., storeName=0af038d6aa643579358dbb0f202b0f57/C, priority=11, startTime=1731192148359; duration=0sec 2024-11-09T22:42:29,216 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T22:42:29,216 DEBUG [RS:0;9d8c37854f6e:35101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0af038d6aa643579358dbb0f202b0f57:C 2024-11-09T22:42:33,340 INFO [Thread-2171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2171. 2024-11-09T22:42:33,340 DEBUG [Thread-2171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:33,340 DEBUG [Thread-2171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:33,340 DEBUG [Thread-2171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:33,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:33,410 INFO [Thread-2173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2173. 2024-11-09T22:42:33,410 DEBUG [Thread-2173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T22:42:33,410 DEBUG [Thread-2173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:33,410 DEBUG [Thread-2173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:33,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:33,410 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4101 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4033 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3914 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4085 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4031 2024-11-09T22:42:33,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T22:42:33,411 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:42:33,412 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:42:33,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:33,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:33,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:33,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T22:42:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T22:42:33,419 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192153418"}]},"ts":"1731192153418"} 2024-11-09T22:42:33,421 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T22:42:33,421 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T22:42:33,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T22:42:33,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, UNASSIGN}] 2024-11-09T22:42:33,425 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, UNASSIGN 2024-11-09T22:42:33,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=0af038d6aa643579358dbb0f202b0f57, regionState=CLOSING, regionLocation=9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:33,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, UNASSIGN because future has completed 2024-11-09T22:42:33,429 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T22:42:33,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027}] 2024-11-09T22:42:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T22:42:33,583 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:33,583 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T22:42:33,583 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 0af038d6aa643579358dbb0f202b0f57, disabling compactions & flushes 2024-11-09T22:42:33,583 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:33,583 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:33,583 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. after waiting 0 ms 2024-11-09T22:42:33,583 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:33,583 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 0af038d6aa643579358dbb0f202b0f57 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T22:42:33,591 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097b9169da60914ba5a0ec48e1a7df89dc_0af038d6aa643579358dbb0f202b0f57 is 50, key is test_row_0/A:col10/1731192153407/Put/seqid=0 2024-11-09T22:42:33,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742387_1563 (size=12454) 2024-11-09T22:42:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T22:42:33,997 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T22:42:34,007 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097b9169da60914ba5a0ec48e1a7df89dc_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b9169da60914ba5a0ec48e1a7df89dc_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:34,008 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/3124b88e882047ea9d240644dc2c0e7c, store: [table=TestAcidGuarantees family=A region=0af038d6aa643579358dbb0f202b0f57] 2024-11-09T22:42:34,008 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/3124b88e882047ea9d240644dc2c0e7c is 175, key is test_row_0/A:col10/1731192153407/Put/seqid=0 2024-11-09T22:42:34,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742388_1564 (size=31255) 2024-11-09T22:42:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T22:42:34,414 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/3124b88e882047ea9d240644dc2c0e7c 2024-11-09T22:42:34,427 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/30b6a2643cfc4c12b85187d06ca45c44 is 50, key is test_row_0/B:col10/1731192153407/Put/seqid=0 2024-11-09T22:42:34,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742389_1565 (size=12301) 2024-11-09T22:42:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T22:42:34,831 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/30b6a2643cfc4c12b85187d06ca45c44 2024-11-09T22:42:34,837 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/42165b9c1da44aa588d6811c1fc388c9 is 50, key is test_row_0/C:col10/1731192153407/Put/seqid=0 2024-11-09T22:42:34,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742390_1566 (size=12301) 2024-11-09T22:42:35,242 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/42165b9c1da44aa588d6811c1fc388c9 2024-11-09T22:42:35,253 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/A/3124b88e882047ea9d240644dc2c0e7c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/3124b88e882047ea9d240644dc2c0e7c 2024-11-09T22:42:35,258 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/3124b88e882047ea9d240644dc2c0e7c, entries=150, sequenceid=299, filesize=30.5 K 2024-11-09T22:42:35,258 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/B/30b6a2643cfc4c12b85187d06ca45c44 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/30b6a2643cfc4c12b85187d06ca45c44 2024-11-09T22:42:35,262 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/30b6a2643cfc4c12b85187d06ca45c44, entries=150, sequenceid=299, filesize=12.0 K 2024-11-09T22:42:35,263 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/.tmp/C/42165b9c1da44aa588d6811c1fc388c9 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/42165b9c1da44aa588d6811c1fc388c9 2024-11-09T22:42:35,266 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/42165b9c1da44aa588d6811c1fc388c9, entries=150, sequenceid=299, filesize=12.0 K 2024-11-09T22:42:35,267 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0af038d6aa643579358dbb0f202b0f57 in 1684ms, sequenceid=299, compaction requested=false 2024-11-09T22:42:35,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/07faa4a7068243c29e9ba0b670d483ac, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/38ea7e326d1d4ef8a5d1e50845ef4773, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/b14239b40d8647ca96e2889f818275c4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/81a519f34c504a4682687c56d4220d14, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/347c6a2812704108ae424b792c3ad331, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/320011209020494395f26dd5061f4968, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/a4a59ee4e39c48f3abeb6f2f3818dfb4, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/32e3a83fe66b4ce2a8f367d5133b8321, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/f83d98ecf0734c4aadbdcf010e8be7d3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/ae5c9086edc54143ba7823ed0c7d37af, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/2ebfe6e8c86141df9bb8deb7abf93787, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/4651af3662274d5985b27018dc7189b6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/0eff74cd9ea44824b528e83bd9ac25db, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/755ce70a10db48d3a6f646ad7f1b5559, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/95ee4884c77447f8a0755cd0face6cff, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/55e2d35544da4532b995280c5024622a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/46cc709a88fc4f4d9d596672df3f5503, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/49df6dae045446d6b24f0cdd4b454a7e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/99ea495252234ab68b1ddba3b3046665] to archive 2024-11-09T22:42:35,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:42:35,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/07faa4a7068243c29e9ba0b670d483ac to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/07faa4a7068243c29e9ba0b670d483ac 2024-11-09T22:42:35,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/38ea7e326d1d4ef8a5d1e50845ef4773 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/38ea7e326d1d4ef8a5d1e50845ef4773 2024-11-09T22:42:35,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/b14239b40d8647ca96e2889f818275c4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/b14239b40d8647ca96e2889f818275c4 2024-11-09T22:42:35,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/81a519f34c504a4682687c56d4220d14 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/81a519f34c504a4682687c56d4220d14 2024-11-09T22:42:35,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/347c6a2812704108ae424b792c3ad331 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/347c6a2812704108ae424b792c3ad331 2024-11-09T22:42:35,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/320011209020494395f26dd5061f4968 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/320011209020494395f26dd5061f4968 2024-11-09T22:42:35,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/a4a59ee4e39c48f3abeb6f2f3818dfb4 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/a4a59ee4e39c48f3abeb6f2f3818dfb4 2024-11-09T22:42:35,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/32e3a83fe66b4ce2a8f367d5133b8321 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/32e3a83fe66b4ce2a8f367d5133b8321 2024-11-09T22:42:35,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/f83d98ecf0734c4aadbdcf010e8be7d3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/f83d98ecf0734c4aadbdcf010e8be7d3 2024-11-09T22:42:35,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/ae5c9086edc54143ba7823ed0c7d37af to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/ae5c9086edc54143ba7823ed0c7d37af 2024-11-09T22:42:35,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/2ebfe6e8c86141df9bb8deb7abf93787 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/2ebfe6e8c86141df9bb8deb7abf93787 2024-11-09T22:42:35,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/4651af3662274d5985b27018dc7189b6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/4651af3662274d5985b27018dc7189b6 2024-11-09T22:42:35,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/0eff74cd9ea44824b528e83bd9ac25db to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/0eff74cd9ea44824b528e83bd9ac25db 2024-11-09T22:42:35,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/755ce70a10db48d3a6f646ad7f1b5559 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/755ce70a10db48d3a6f646ad7f1b5559 2024-11-09T22:42:35,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/95ee4884c77447f8a0755cd0face6cff to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/95ee4884c77447f8a0755cd0face6cff 2024-11-09T22:42:35,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/55e2d35544da4532b995280c5024622a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/55e2d35544da4532b995280c5024622a 2024-11-09T22:42:35,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/46cc709a88fc4f4d9d596672df3f5503 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/46cc709a88fc4f4d9d596672df3f5503 2024-11-09T22:42:35,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/49df6dae045446d6b24f0cdd4b454a7e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/49df6dae045446d6b24f0cdd4b454a7e 2024-11-09T22:42:35,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/99ea495252234ab68b1ddba3b3046665 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/99ea495252234ab68b1ddba3b3046665 2024-11-09T22:42:35,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/47822e5f8c9f404b8829bae9680ec5c8, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96013d2d490d4f4298a732418e510f4a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/bcfec35b8ce04835aedcdca6c56d5f1c, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c0fda3840c8b40319f361b9d60d0c50b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96bb8790d4874382b6c5db27d9d1f749, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4d4f8d76e15d464780b84538b1fdf4bb, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/efe8975b713042c2b515540659c923d3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4a4295d5b0b94708b5c51fa0e1b93c2a, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/2f9ef7266a0046ab8b59f77c3db70805, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/6e5e4fca36684f91b7940a9bf4e92f2d, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/21ddd042428a4113b165b52db953c533, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/172a8792d13642cf8b79811677774db2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/57e20006f81c4078bbbff9c639dfcb3e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/65cdc5cd892f47d9a44d6825443f15be, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c6cb3e6b2e2747a4b3ed097b880d820b, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/f6933c40e21e414bac456463b6ff5048, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/53c94092944242d2941dfbf8ee0ca1b0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/92fcd8f44b854cb7acb41ea231a66ad6, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/be4b57778c28499b9ace9eb1f96758f6] to archive 2024-11-09T22:42:35,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:42:35,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/47822e5f8c9f404b8829bae9680ec5c8 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/47822e5f8c9f404b8829bae9680ec5c8 2024-11-09T22:42:35,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96013d2d490d4f4298a732418e510f4a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96013d2d490d4f4298a732418e510f4a 2024-11-09T22:42:35,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/bcfec35b8ce04835aedcdca6c56d5f1c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/bcfec35b8ce04835aedcdca6c56d5f1c 2024-11-09T22:42:35,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c0fda3840c8b40319f361b9d60d0c50b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c0fda3840c8b40319f361b9d60d0c50b 2024-11-09T22:42:35,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96bb8790d4874382b6c5db27d9d1f749 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/96bb8790d4874382b6c5db27d9d1f749 2024-11-09T22:42:35,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4d4f8d76e15d464780b84538b1fdf4bb to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4d4f8d76e15d464780b84538b1fdf4bb 2024-11-09T22:42:35,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/efe8975b713042c2b515540659c923d3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/efe8975b713042c2b515540659c923d3 2024-11-09T22:42:35,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4a4295d5b0b94708b5c51fa0e1b93c2a to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/4a4295d5b0b94708b5c51fa0e1b93c2a 2024-11-09T22:42:35,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/2f9ef7266a0046ab8b59f77c3db70805 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/2f9ef7266a0046ab8b59f77c3db70805 2024-11-09T22:42:35,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/6e5e4fca36684f91b7940a9bf4e92f2d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/6e5e4fca36684f91b7940a9bf4e92f2d 2024-11-09T22:42:35,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/21ddd042428a4113b165b52db953c533 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/21ddd042428a4113b165b52db953c533 2024-11-09T22:42:35,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/172a8792d13642cf8b79811677774db2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/172a8792d13642cf8b79811677774db2 2024-11-09T22:42:35,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/57e20006f81c4078bbbff9c639dfcb3e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/57e20006f81c4078bbbff9c639dfcb3e 2024-11-09T22:42:35,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/65cdc5cd892f47d9a44d6825443f15be to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/65cdc5cd892f47d9a44d6825443f15be 2024-11-09T22:42:35,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c6cb3e6b2e2747a4b3ed097b880d820b to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/c6cb3e6b2e2747a4b3ed097b880d820b 2024-11-09T22:42:35,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/f6933c40e21e414bac456463b6ff5048 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/f6933c40e21e414bac456463b6ff5048 2024-11-09T22:42:35,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/53c94092944242d2941dfbf8ee0ca1b0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/53c94092944242d2941dfbf8ee0ca1b0 2024-11-09T22:42:35,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/92fcd8f44b854cb7acb41ea231a66ad6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/92fcd8f44b854cb7acb41ea231a66ad6 2024-11-09T22:42:35,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/be4b57778c28499b9ace9eb1f96758f6 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/be4b57778c28499b9ace9eb1f96758f6 2024-11-09T22:42:35,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/4f484f5faafa4ad3bb3ce90a34cb92fa, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/5266b8b1fe4e4e2ab3f44f21d687d8c9, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/24ffce2873ba49ee8e7db09620a55a43, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/9ce4cf99e28b4a56bb11f83b93d90512, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/29243fdd264b4ff49cc0189bf650a5ea, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/a9c8dbf74307484db2a0d19500f8dbee, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/250e62ee65d941f0be8d32e1aa66da2f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2be8a02f353f479984d8e2378c25ab95, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/12cd4cf692504f008e456831d7731d83, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/d2b6fd723bfc46489a94b34b73b67235, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/3e4b1259e6914c408798f208bdf9d6f1, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/88edd9f1bb7d44c2805238a17e24eef2, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/f048f3a40763468187deb0f7ed5d8c53, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/193f0999062348979f8b572934a53fc3, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/38ee0b15b9f840cdad9c2ef96292a65f, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/359e246d912249f998957e25977bf46e, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/6453ac73477b46b192138c59c6e4f5be, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/ed602da0017046e2997f7478679d5ff0, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/8c58d2a2f45747a9a7c01685f5b993b9] to archive 2024-11-09T22:42:35,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T22:42:35,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/4f484f5faafa4ad3bb3ce90a34cb92fa to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/4f484f5faafa4ad3bb3ce90a34cb92fa 2024-11-09T22:42:35,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/5266b8b1fe4e4e2ab3f44f21d687d8c9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/5266b8b1fe4e4e2ab3f44f21d687d8c9 2024-11-09T22:42:35,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/24ffce2873ba49ee8e7db09620a55a43 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/24ffce2873ba49ee8e7db09620a55a43 2024-11-09T22:42:35,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/9ce4cf99e28b4a56bb11f83b93d90512 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/9ce4cf99e28b4a56bb11f83b93d90512 2024-11-09T22:42:35,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/29243fdd264b4ff49cc0189bf650a5ea to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/29243fdd264b4ff49cc0189bf650a5ea 2024-11-09T22:42:35,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/a9c8dbf74307484db2a0d19500f8dbee to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/a9c8dbf74307484db2a0d19500f8dbee 2024-11-09T22:42:35,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/250e62ee65d941f0be8d32e1aa66da2f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/250e62ee65d941f0be8d32e1aa66da2f 2024-11-09T22:42:35,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2be8a02f353f479984d8e2378c25ab95 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2be8a02f353f479984d8e2378c25ab95 2024-11-09T22:42:35,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/12cd4cf692504f008e456831d7731d83 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/12cd4cf692504f008e456831d7731d83 2024-11-09T22:42:35,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/d2b6fd723bfc46489a94b34b73b67235 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/d2b6fd723bfc46489a94b34b73b67235 2024-11-09T22:42:35,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/3e4b1259e6914c408798f208bdf9d6f1 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/3e4b1259e6914c408798f208bdf9d6f1 2024-11-09T22:42:35,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/88edd9f1bb7d44c2805238a17e24eef2 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/88edd9f1bb7d44c2805238a17e24eef2 2024-11-09T22:42:35,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/f048f3a40763468187deb0f7ed5d8c53 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/f048f3a40763468187deb0f7ed5d8c53 2024-11-09T22:42:35,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/193f0999062348979f8b572934a53fc3 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/193f0999062348979f8b572934a53fc3 2024-11-09T22:42:35,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/38ee0b15b9f840cdad9c2ef96292a65f to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/38ee0b15b9f840cdad9c2ef96292a65f 2024-11-09T22:42:35,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/359e246d912249f998957e25977bf46e to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/359e246d912249f998957e25977bf46e 2024-11-09T22:42:35,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/6453ac73477b46b192138c59c6e4f5be to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/6453ac73477b46b192138c59c6e4f5be 2024-11-09T22:42:35,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/ed602da0017046e2997f7478679d5ff0 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/ed602da0017046e2997f7478679d5ff0 2024-11-09T22:42:35,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/8c58d2a2f45747a9a7c01685f5b993b9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/8c58d2a2f45747a9a7c01685f5b993b9 2024-11-09T22:42:35,313 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/recovered.edits/302.seqid, newMaxSeqId=302, maxSeqId=4 2024-11-09T22:42:35,313 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. 2024-11-09T22:42:35,313 DEBUG [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 0af038d6aa643579358dbb0f202b0f57: Waiting for close lock at 1731192153583Running coprocessor pre-close hooks at 1731192153583Disabling compacts and flushes for region at 1731192153583Disabling writes for close at 1731192153583Obtaining lock to block concurrent updates at 1731192153583Preparing flush snapshotting stores in 0af038d6aa643579358dbb0f202b0f57 at 1731192153583Finished memstore snapshotting TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731192153584 (+1 ms)Flushing stores of TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57. at 1731192153585 (+1 ms)Flushing 0af038d6aa643579358dbb0f202b0f57/A: creating writer at 1731192153585Flushing mob file 0af038d6aa643579358dbb0f202b0f57/A: appending metadata at 1731192153591 (+6 ms)Flushing mob file 0af038d6aa643579358dbb0f202b0f57/A: closing flushed file at 1731192153591Flushing 0af038d6aa643579358dbb0f202b0f57/A: appending metadata at 1731192154008 (+417 ms)Flushing 0af038d6aa643579358dbb0f202b0f57/A: closing flushed file at 1731192154008Flushing 0af038d6aa643579358dbb0f202b0f57/B: creating writer at 1731192154424 (+416 ms)Flushing 0af038d6aa643579358dbb0f202b0f57/B: appending metadata at 1731192154427 (+3 ms)Flushing 0af038d6aa643579358dbb0f202b0f57/B: closing flushed file at 1731192154427Flushing 0af038d6aa643579358dbb0f202b0f57/C: creating writer at 1731192154834 (+407 ms)Flushing 0af038d6aa643579358dbb0f202b0f57/C: appending metadata at 1731192154837 (+3 ms)Flushing 0af038d6aa643579358dbb0f202b0f57/C: closing flushed file at 1731192154837Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63fc87cb: reopening flushed file at 1731192155252 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76c3f169: reopening flushed file at 1731192155258 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7081442c: reopening flushed file at 1731192155262 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0af038d6aa643579358dbb0f202b0f57 in 1684ms, sequenceid=299, compaction requested=false at 1731192155267 (+5 ms)Writing region close event to WAL at 1731192155311 (+44 ms)Running coprocessor post-close hooks at 1731192155313 (+2 ms)Closed at 1731192155313 2024-11-09T22:42:35,315 INFO [RS_CLOSE_REGION-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,315 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=0af038d6aa643579358dbb0f202b0f57, regionState=CLOSED 2024-11-09T22:42:35,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 because future has completed 2024-11-09T22:42:35,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-09T22:42:35,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 0af038d6aa643579358dbb0f202b0f57, server=9d8c37854f6e,35101,1731191981027 in 1.8880 sec 2024-11-09T22:42:35,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-09T22:42:35,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0af038d6aa643579358dbb0f202b0f57, UNASSIGN in 1.8950 sec 2024-11-09T22:42:35,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-09T22:42:35,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8990 sec 2024-11-09T22:42:35,324 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731192155323"}]},"ts":"1731192155323"} 2024-11-09T22:42:35,325 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T22:42:35,325 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T22:42:35,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9120 sec 2024-11-09T22:42:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T22:42:35,567 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:35,568 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T22:42:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:35,573 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T22:42:35,574 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:35,579 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,583 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C, FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/recovered.edits] 2024-11-09T22:42:35,588 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/3124b88e882047ea9d240644dc2c0e7c to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/3124b88e882047ea9d240644dc2c0e7c 2024-11-09T22:42:35,591 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/d340a2343d5e411385df612073e65c32 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/A/d340a2343d5e411385df612073e65c32 2024-11-09T22:42:35,593 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/30b6a2643cfc4c12b85187d06ca45c44 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/30b6a2643cfc4c12b85187d06ca45c44 2024-11-09T22:42:35,594 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/b7e6a25d96b048cc8b9d6e3d5422c575 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/B/b7e6a25d96b048cc8b9d6e3d5422c575 2024-11-09T22:42:35,596 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2d927d5182f842bbaf23912d2dd5ac2d to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/2d927d5182f842bbaf23912d2dd5ac2d 2024-11-09T22:42:35,597 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/42165b9c1da44aa588d6811c1fc388c9 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/C/42165b9c1da44aa588d6811c1fc388c9 2024-11-09T22:42:35,599 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/recovered.edits/302.seqid to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57/recovered.edits/302.seqid 2024-11-09T22:42:35,599 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/default/TestAcidGuarantees/0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,599 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T22:42:35,599 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T22:42:35,600 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T22:42:35,602 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110900c05e5fa22f47c5949e9d123676b96b_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110900c05e5fa22f47c5949e9d123676b96b_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,604 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090feb647ee1364ae690a9055aca9fe7ad_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090feb647ee1364ae690a9055aca9fe7ad_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,605 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091594221c68e14cfb919e94dc8d416189_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091594221c68e14cfb919e94dc8d416189_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,605 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091879a547e1ca4dd7810821cd4ea8c57c_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091879a547e1ca4dd7810821cd4ea8c57c_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,606 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110924e42b4accd34974bfa5973a282c8046_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110924e42b4accd34974bfa5973a282c8046_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,607 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110935595c32fa7d413eabff2d32ccb89140_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110935595c32fa7d413eabff2d32ccb89140_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,608 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963ec61807a29429188a5eac59620d879_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110963ec61807a29429188a5eac59620d879_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,609 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b9169da60914ba5a0ec48e1a7df89dc_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097b9169da60914ba5a0ec48e1a7df89dc_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,610 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110990121b188db14e7b9c642b00cfee6ae6_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110990121b188db14e7b9c642b00cfee6ae6_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,611 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b55ee2ccccd841289de13aac87fe410b_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b55ee2ccccd841289de13aac87fe410b_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,612 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b8f12372eb2b4271acc9d03ffdf97604_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b8f12372eb2b4271acc9d03ffdf97604_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,613 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109bf48bff627e54993b6b2f1ab6498947e_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109bf48bff627e54993b6b2f1ab6498947e_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,614 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c5d5b8ef213143559c7299fe04145ade_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c5d5b8ef213143559c7299fe04145ade_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,616 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c71978034ffd4ad4b7492f13fb77d7df_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c71978034ffd4ad4b7492f13fb77d7df_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,617 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dfc4ad44fc084fc7a6a64e355dcca575_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dfc4ad44fc084fc7a6a64e355dcca575_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,619 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ffeb228fad7b4ee683977ce0c32b21ac_0af038d6aa643579358dbb0f202b0f57 to hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ffeb228fad7b4ee683977ce0c32b21ac_0af038d6aa643579358dbb0f202b0f57 2024-11-09T22:42:35,620 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T22:42:35,622 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:35,625 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T22:42:35,627 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T22:42:35,629 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:35,629 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T22:42:35,629 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731192155629"}]},"ts":"9223372036854775807"} 2024-11-09T22:42:35,631 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T22:42:35,631 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0af038d6aa643579358dbb0f202b0f57, NAME => 'TestAcidGuarantees,,1731192121872.0af038d6aa643579358dbb0f202b0f57.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T22:42:35,631 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T22:42:35,631 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731192155631"}]},"ts":"9223372036854775807"} 2024-11-09T22:42:35,632 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T22:42:35,633 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T22:42:35,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-09T22:42:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34651 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T22:42:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T22:42:35,686 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T22:42:35,699 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238 (was 240), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 436), ProcessCount=11 (was 11), AvailableMemoryMB=17609 (was 17620) 2024-11-09T22:42:35,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T22:42:35,699 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:42:35,699 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:42:35,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:35,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:35,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T22:42:35,700 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T22:42:35,700 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2051383393, stopped=false 2024-11-09T22:42:35,700 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9d8c37854f6e,34651,1731191980160 2024-11-09T22:42:35,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T22:42:35,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T22:42:35,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:42:35,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:42:35,760 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T22:42:35,762 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T22:42:35,762 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:42:35,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:35,762 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T22:42:35,763 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T22:42:35,764 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9d8c37854f6e,35101,1731191981027' ***** 2024-11-09T22:42:35,764 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T22:42:35,764 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T22:42:35,765 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T22:42:35,765 INFO [RS:0;9d8c37854f6e:35101 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T22:42:35,766 INFO [RS:0;9d8c37854f6e:35101 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T22:42:35,766 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(959): stopping server 9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:35,766 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T22:42:35,766 INFO [RS:0;9d8c37854f6e:35101 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9d8c37854f6e:35101. 2024-11-09T22:42:35,766 DEBUG [RS:0;9d8c37854f6e:35101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T22:42:35,766 DEBUG [RS:0;9d8c37854f6e:35101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:35,767 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T22:42:35,767 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T22:42:35,767 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T22:42:35,767 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T22:42:35,768 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-09T22:42:35,768 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-09T22:42:35,768 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T22:42:35,768 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T22:42:35,768 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:35,768 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T22:42:35,768 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T22:42:35,768 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T22:42:35,768 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-09T22:42:35,789 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/info/32fdef6314344ea79bce174261fca211 is 95, key is TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16./info:/1731192011512/DeleteFamily/seqid=0 2024-11-09T22:42:35,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742391_1567 (size=6514) 2024-11-09T22:42:35,856 INFO [regionserver/9d8c37854f6e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T22:42:35,899 INFO [regionserver/9d8c37854f6e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T22:42:35,899 INFO [regionserver/9d8c37854f6e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T22:42:35,968 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:36,168 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:36,194 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/info/32fdef6314344ea79bce174261fca211 2024-11-09T22:42:36,220 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/ns/e3002bba83b64d219d3dcba12ed8bbe8 is 93, key is TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16./ns:/1731192011472/DeleteFamily/seqid=0 2024-11-09T22:42:36,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742392_1568 (size=5906) 2024-11-09T22:42:36,369 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:36,569 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:36,624 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/ns/e3002bba83b64d219d3dcba12ed8bbe8 2024-11-09T22:42:36,653 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/rep_barrier/7c8c792b5ece4e2c8f207ce754dfcd8c is 102, key is TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16./rep_barrier:/1731192011472/DeleteFamily/seqid=0 2024-11-09T22:42:36,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742393_1569 (size=6025) 2024-11-09T22:42:36,769 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:36,970 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:37,058 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/rep_barrier/7c8c792b5ece4e2c8f207ce754dfcd8c 2024-11-09T22:42:37,081 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/table/c1750218f7d24f1abcd5d570903b455a is 96, key is TestAcidGuarantees,,1731191984983.1f18320792a575fd182c9198939e0c16./table:/1731192011472/DeleteFamily/seqid=0 2024-11-09T22:42:37,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742394_1570 (size=5936) 2024-11-09T22:42:37,170 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:37,370 DEBUG [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T22:42:37,485 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/table/c1750218f7d24f1abcd5d570903b455a 2024-11-09T22:42:37,496 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/info/32fdef6314344ea79bce174261fca211 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/info/32fdef6314344ea79bce174261fca211 2024-11-09T22:42:37,500 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/info/32fdef6314344ea79bce174261fca211, entries=12, sequenceid=90, filesize=6.4 K 2024-11-09T22:42:37,501 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/ns/e3002bba83b64d219d3dcba12ed8bbe8 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/ns/e3002bba83b64d219d3dcba12ed8bbe8 2024-11-09T22:42:37,504 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/ns/e3002bba83b64d219d3dcba12ed8bbe8, entries=8, sequenceid=90, filesize=5.8 K 2024-11-09T22:42:37,505 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/rep_barrier/7c8c792b5ece4e2c8f207ce754dfcd8c as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/rep_barrier/7c8c792b5ece4e2c8f207ce754dfcd8c 2024-11-09T22:42:37,508 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/rep_barrier/7c8c792b5ece4e2c8f207ce754dfcd8c, entries=6, sequenceid=90, filesize=5.9 K 2024-11-09T22:42:37,508 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/.tmp/table/c1750218f7d24f1abcd5d570903b455a as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/table/c1750218f7d24f1abcd5d570903b455a 2024-11-09T22:42:37,512 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/table/c1750218f7d24f1abcd5d570903b455a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-09T22:42:37,512 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1744ms, sequenceid=90, compaction requested=false 2024-11-09T22:42:37,517 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-09T22:42:37,518 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T22:42:37,518 INFO [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T22:42:37,518 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731192155768Running coprocessor pre-close hooks at 1731192155768Disabling compacts and flushes for region at 1731192155768Disabling writes for close at 1731192155768Obtaining lock to block concurrent updates at 1731192155768Preparing flush snapshotting stores in 1588230740 at 1731192155768Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731192155769 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731192155769Flushing 1588230740/info: creating writer at 1731192155769Flushing 1588230740/info: appending metadata at 1731192155789 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731192155789Flushing 1588230740/ns: creating writer at 1731192156204 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731192156220 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731192156220Flushing 1588230740/rep_barrier: creating writer at 1731192156634 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731192156653 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731192156653Flushing 1588230740/table: creating writer at 1731192157065 (+412 ms)Flushing 1588230740/table: appending metadata at 1731192157080 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731192157080Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d2d0bce: reopening flushed file at 1731192157495 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c2f705f: reopening flushed file at 1731192157500 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ecebd59: reopening flushed file at 1731192157504 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34b74778: reopening flushed file at 1731192157508 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1744ms, sequenceid=90, compaction requested=false at 1731192157512 (+4 ms)Writing region close event to WAL at 1731192157514 (+2 ms)Running coprocessor post-close hooks at 1731192157518 (+4 ms)Closed at 1731192157518 2024-11-09T22:42:37,518 DEBUG [RS_CLOSE_META-regionserver/9d8c37854f6e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T22:42:37,570 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(976): stopping server 9d8c37854f6e,35101,1731191981027; all regions closed. 2024-11-09T22:42:37,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741834_1010 (size=25344) 2024-11-09T22:42:37,576 DEBUG [RS:0;9d8c37854f6e:35101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/oldWALs 2024-11-09T22:42:37,577 INFO [RS:0;9d8c37854f6e:35101 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9d8c37854f6e%2C35101%2C1731191981027.meta:.meta(num 1731191984493) 2024-11-09T22:42:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741833_1009 (size=12905940) 2024-11-09T22:42:37,581 DEBUG [RS:0;9d8c37854f6e:35101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/oldWALs 2024-11-09T22:42:37,581 INFO [RS:0;9d8c37854f6e:35101 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9d8c37854f6e%2C35101%2C1731191981027:(num 1731191984000) 2024-11-09T22:42:37,581 DEBUG [RS:0;9d8c37854f6e:35101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T22:42:37,581 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T22:42:37,581 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T22:42:37,581 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.ChoreService(370): Chore service for: regionserver/9d8c37854f6e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T22:42:37,581 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T22:42:37,581 INFO [regionserver/9d8c37854f6e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T22:42:37,582 INFO [RS:0;9d8c37854f6e:35101 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35101 2024-11-09T22:42:37,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T22:42:37,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9d8c37854f6e,35101,1731191981027 2024-11-09T22:42:37,634 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T22:42:37,642 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9d8c37854f6e,35101,1731191981027] 2024-11-09T22:42:37,651 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9d8c37854f6e,35101,1731191981027 already deleted, retry=false 2024-11-09T22:42:37,651 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9d8c37854f6e,35101,1731191981027 expired; onlineServers=0 2024-11-09T22:42:37,651 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9d8c37854f6e,34651,1731191980160' ***** 2024-11-09T22:42:37,651 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T22:42:37,651 INFO [M:0;9d8c37854f6e:34651 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T22:42:37,651 INFO [M:0;9d8c37854f6e:34651 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T22:42:37,651 DEBUG [M:0;9d8c37854f6e:34651 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T22:42:37,652 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T22:42:37,652 DEBUG [M:0;9d8c37854f6e:34651 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T22:42:37,652 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster-HFileCleaner.small.0-1731191983629 {}] cleaner.HFileCleaner(306): Exit Thread[master/9d8c37854f6e:0:becomeActiveMaster-HFileCleaner.small.0-1731191983629,5,FailOnTimeoutGroup] 2024-11-09T22:42:37,652 DEBUG [master/9d8c37854f6e:0:becomeActiveMaster-HFileCleaner.large.0-1731191983629 {}] cleaner.HFileCleaner(306): Exit Thread[master/9d8c37854f6e:0:becomeActiveMaster-HFileCleaner.large.0-1731191983629,5,FailOnTimeoutGroup] 2024-11-09T22:42:37,652 INFO [M:0;9d8c37854f6e:34651 {}] hbase.ChoreService(370): Chore service for: master/9d8c37854f6e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T22:42:37,652 INFO [M:0;9d8c37854f6e:34651 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T22:42:37,652 DEBUG [M:0;9d8c37854f6e:34651 {}] master.HMaster(1795): Stopping service threads 2024-11-09T22:42:37,652 INFO [M:0;9d8c37854f6e:34651 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T22:42:37,652 INFO [M:0;9d8c37854f6e:34651 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T22:42:37,652 ERROR [M:0;9d8c37854f6e:34651 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:46069 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46069,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-09T22:42:37,653 INFO [M:0;9d8c37854f6e:34651 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T22:42:37,653 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T22:42:37,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T22:42:37,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T22:42:37,659 DEBUG [M:0;9d8c37854f6e:34651 {}] zookeeper.ZKUtil(347): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T22:42:37,660 WARN [M:0;9d8c37854f6e:34651 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T22:42:37,660 INFO [M:0;9d8c37854f6e:34651 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/.lastflushedseqids 2024-11-09T22:42:37,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742395_1571 (size=72) 2024-11-09T22:42:37,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T22:42:37,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35101-0x101230acffd0001, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T22:42:37,743 INFO [RS:0;9d8c37854f6e:35101 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T22:42:37,744 INFO [RS:0;9d8c37854f6e:35101 {}] regionserver.HRegionServer(1031): Exiting; stopping=9d8c37854f6e,35101,1731191981027; zookeeper connection closed. 2024-11-09T22:42:37,744 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@719a7007 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@719a7007 2024-11-09T22:42:37,745 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T22:42:38,073 INFO [M:0;9d8c37854f6e:34651 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T22:42:38,074 INFO [M:0;9d8c37854f6e:34651 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T22:42:38,074 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T22:42:38,074 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:42:38,074 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:42:38,075 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T22:42:38,075 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:42:38,075 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=718.31 KB heapSize=879.97 KB 2024-11-09T22:42:38,096 DEBUG [M:0;9d8c37854f6e:34651 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb5f1bb5c6f347249482232a17bca4b7 is 82, key is hbase:meta,,1/info:regioninfo/1731191984584/Put/seqid=0 2024-11-09T22:42:38,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742396_1572 (size=5672) 2024-11-09T22:42:38,500 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1985 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb5f1bb5c6f347249482232a17bca4b7 2024-11-09T22:42:38,535 DEBUG [M:0;9d8c37854f6e:34651 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88c70f72fabc4180b8b840c714cb2cc0 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1731192124995/Put/seqid=0 2024-11-09T22:42:38,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742397_1573 (size=43431) 2024-11-09T22:42:38,939 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717.75 KB at sequenceid=1985 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88c70f72fabc4180b8b840c714cb2cc0 2024-11-09T22:42:38,945 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 88c70f72fabc4180b8b840c714cb2cc0 2024-11-09T22:42:38,964 DEBUG [M:0;9d8c37854f6e:34651 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5829b4de8ff64e35a9839123687c2b41 is 69, key is 9d8c37854f6e,35101,1731191981027/rs:state/1731191983709/Put/seqid=0 2024-11-09T22:42:38,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073742398_1574 (size=5156) 2024-11-09T22:42:39,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T22:42:39,368 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1985 (bloomFilter=true), to=hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5829b4de8ff64e35a9839123687c2b41 2024-11-09T22:42:39,380 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb5f1bb5c6f347249482232a17bca4b7 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eb5f1bb5c6f347249482232a17bca4b7 2024-11-09T22:42:39,384 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eb5f1bb5c6f347249482232a17bca4b7, entries=8, sequenceid=1985, filesize=5.5 K 2024-11-09T22:42:39,384 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/88c70f72fabc4180b8b840c714cb2cc0 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/88c70f72fabc4180b8b840c714cb2cc0 2024-11-09T22:42:39,388 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 88c70f72fabc4180b8b840c714cb2cc0 2024-11-09T22:42:39,388 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/88c70f72fabc4180b8b840c714cb2cc0, entries=166, sequenceid=1985, filesize=42.4 K 2024-11-09T22:42:39,388 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5829b4de8ff64e35a9839123687c2b41 as hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5829b4de8ff64e35a9839123687c2b41 2024-11-09T22:42:39,392 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46069/user/jenkins/test-data/fd6824e5-d80a-1cfd-a9e8-63f9f0508696/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5829b4de8ff64e35a9839123687c2b41, entries=1, sequenceid=1985, filesize=5.0 K 2024-11-09T22:42:39,393 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(3140): Finished flush of dataSize ~718.31 KB/735547, heapSize ~879.67 KB/900784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1318ms, sequenceid=1985, compaction requested=false 2024-11-09T22:42:39,394 INFO [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T22:42:39,394 DEBUG [M:0;9d8c37854f6e:34651 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731192158074Disabling compacts and flushes for region at 1731192158074Disabling writes for close at 1731192158075 (+1 ms)Obtaining lock to block concurrent updates at 1731192158075Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731192158075Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=735547, getHeapSize=901024, getOffHeapSize=0, getCellsCount=2176 at 1731192158076 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731192158077 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731192158077Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731192158095 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731192158095Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731192158509 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731192158535 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731192158535Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731192158946 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731192158963 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731192158963Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30ad3bc5: reopening flushed file at 1731192159379 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c2556a: reopening flushed file at 1731192159384 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63583b45: reopening flushed file at 1731192159388 (+4 ms)Finished flush of dataSize ~718.31 KB/735547, heapSize ~879.67 KB/900784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1318ms, sequenceid=1985, compaction requested=false at 1731192159393 (+5 ms)Writing region close event to WAL at 1731192159394 (+1 ms)Closed at 1731192159394 2024-11-09T22:42:39,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741830_1006 (size=865755) 2024-11-09T22:42:39,398 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T22:42:39,398 INFO [M:0;9d8c37854f6e:34651 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T22:42:39,398 INFO [M:0;9d8c37854f6e:34651 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34651 2024-11-09T22:42:39,398 INFO [M:0;9d8c37854f6e:34651 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T22:42:39,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T22:42:39,551 INFO [M:0;9d8c37854f6e:34651 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T22:42:39,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34651-0x101230acffd0000, quorum=127.0.0.1:49842, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T22:42:39,557 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T22:42:39,560 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T22:42:39,560 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T22:42:39,560 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T22:42:39,560 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/hadoop.log.dir/,STOPPED} 2024-11-09T22:42:39,563 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T22:42:39,563 WARN [BP-454095681-172.17.0.2-1731191976142 heartbeating to localhost/127.0.0.1:46069 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T22:42:39,563 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T22:42:39,563 WARN [BP-454095681-172.17.0.2-1731191976142 heartbeating to localhost/127.0.0.1:46069 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-454095681-172.17.0.2-1731191976142 (Datanode Uuid 67278256-a007-410b-a3ce-fada866abf51) service to localhost/127.0.0.1:46069 2024-11-09T22:42:39,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/data/data1/current/BP-454095681-172.17.0.2-1731191976142 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T22:42:39,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/cluster_99c618ec-8bdc-7e63-4d7c-fff1881394a5/data/data2/current/BP-454095681-172.17.0.2-1731191976142 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T22:42:39,567 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T22:42:39,576 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T22:42:39,576 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T22:42:39,576 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T22:42:39,576 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T22:42:39,577 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21aa203c-61b1-d347-7290-c4f5a08f1f09/hadoop.log.dir/,STOPPED} 2024-11-09T22:42:39,591 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T22:42:39,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down